| Line Number |
../DebugInfoTest/example_mips_dbg.ll
|
Hit count |
Line Number |
../DebugInfoTest/example_mips.ll
|
Hit count |
| 1 |
//===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===// |
--- |
1 |
//===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===// |
--- |
| 2 |
// |
--- |
2 |
// |
--- |
| 3 |
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
--- |
3 |
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
--- |
| 4 |
// See https://llvm.org/LICENSE.txt for license information. |
--- |
4 |
// See https://llvm.org/LICENSE.txt for license information. |
--- |
| 5 |
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
--- |
5 |
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
--- |
| 6 |
// |
--- |
6 |
// |
--- |
| 7 |
//===----------------------------------------------------------------------===// |
--- |
7 |
//===----------------------------------------------------------------------===// |
--- |
| 8 |
// |
--- |
8 |
// |
--- |
| 9 |
// This file implements integer type expansion and promotion for LegalizeTypes. |
--- |
9 |
// This file implements integer type expansion and promotion for LegalizeTypes. |
--- |
| 10 |
// Promotion is the act of changing a computation in an illegal type into a |
--- |
10 |
// Promotion is the act of changing a computation in an illegal type into a |
--- |
| 11 |
// computation in a larger type. For example, implementing i8 arithmetic in an |
--- |
11 |
// computation in a larger type. For example, implementing i8 arithmetic in an |
--- |
| 12 |
// i32 register (often needed on powerpc). |
--- |
12 |
// i32 register (often needed on powerpc). |
--- |
| 13 |
// Expansion is the act of changing a computation in an illegal type into a |
--- |
13 |
// Expansion is the act of changing a computation in an illegal type into a |
--- |
| 14 |
// computation in two identical registers of a smaller type. For example, |
--- |
14 |
// computation in two identical registers of a smaller type. For example, |
--- |
| 15 |
// implementing i64 arithmetic in two i32 registers (often needed on 32-bit |
--- |
15 |
// implementing i64 arithmetic in two i32 registers (often needed on 32-bit |
--- |
| 16 |
// targets). |
--- |
16 |
// targets). |
--- |
| 17 |
// |
--- |
17 |
// |
--- |
| 18 |
//===----------------------------------------------------------------------===// |
--- |
18 |
//===----------------------------------------------------------------------===// |
--- |
| 19 |
|
--- |
19 |
|
--- |
| 20 |
#include "LegalizeTypes.h" |
--- |
20 |
#include "LegalizeTypes.h" |
--- |
| 21 |
#include "llvm/Analysis/TargetLibraryInfo.h" |
--- |
21 |
#include "llvm/Analysis/TargetLibraryInfo.h" |
--- |
| 22 |
#include "llvm/CodeGen/StackMaps.h" |
--- |
22 |
#include "llvm/CodeGen/StackMaps.h" |
--- |
| 23 |
#include "llvm/CodeGen/TargetLowering.h" |
--- |
23 |
#include "llvm/CodeGen/TargetLowering.h" |
--- |
| 24 |
#include "llvm/IR/DerivedTypes.h" |
--- |
24 |
#include "llvm/IR/DerivedTypes.h" |
--- |
| 25 |
#include "llvm/Support/ErrorHandling.h" |
--- |
25 |
#include "llvm/Support/ErrorHandling.h" |
--- |
| 26 |
#include "llvm/Support/KnownBits.h" |
--- |
26 |
#include "llvm/Support/KnownBits.h" |
--- |
| 27 |
#include "llvm/Support/raw_ostream.h" |
--- |
27 |
#include "llvm/Support/raw_ostream.h" |
--- |
| 28 |
#include |
--- |
28 |
#include |
--- |
| 29 |
using namespace llvm; |
--- |
29 |
using namespace llvm; |
--- |
| 30 |
|
--- |
30 |
|
--- |
| 31 |
#define DEBUG_TYPE "legalize-types" |
--- |
31 |
#define DEBUG_TYPE "legalize-types" |
--- |
| 32 |
|
--- |
32 |
|
--- |
| 33 |
//===----------------------------------------------------------------------===// |
--- |
33 |
//===----------------------------------------------------------------------===// |
--- |
| 34 |
// Integer Result Promotion |
--- |
34 |
// Integer Result Promotion |
--- |
| 35 |
//===----------------------------------------------------------------------===// |
--- |
35 |
//===----------------------------------------------------------------------===// |
--- |
| 36 |
|
--- |
36 |
|
--- |
| 37 |
/// PromoteIntegerResult - This method is called when a result of a node is |
--- |
37 |
/// PromoteIntegerResult - This method is called when a result of a node is |
--- |
| 38 |
/// found to be in need of promotion to a larger type. At this point, the node |
--- |
38 |
/// found to be in need of promotion to a larger type. At this point, the node |
--- |
| 39 |
/// may also have invalid operands or may have other results that need |
--- |
39 |
/// may also have invalid operands or may have other results that need |
--- |
| 40 |
/// expansion, we just know that (at least) one result needs promotion. |
--- |
40 |
/// expansion, we just know that (at least) one result needs promotion. |
--- |
| 41 |
void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { |
1 |
41 |
void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { |
1 |
| 42 |
LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG); |
1 |
42 |
LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG); |
1 |
| 43 |
dbgs() << "\n"); |
--- |
43 |
dbgs() << "\n"); |
--- |
| 44 |
SDValue Res = SDValue(); |
1 |
44 |
SDValue Res = SDValue(); |
1 |
| 45 |
|
--- |
45 |
|
--- |
| 46 |
// See if the target wants to custom expand this node. |
--- |
46 |
// See if the target wants to custom expand this node. |
--- |
| 47 |
if (CustomLowerNode(N, N->getValueType(ResNo), true)) { |
1 |
47 |
if (CustomLowerNode(N, N->getValueType(ResNo), true)) { |
1 |
| 48 |
LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n"); |
0 |
48 |
LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n"); |
0 |
| 49 |
return; |
0 |
49 |
return; |
0 |
| 50 |
} |
--- |
50 |
} |
--- |
| 51 |
|
--- |
51 |
|
--- |
| 52 |
switch (N->getOpcode()) { |
1 |
52 |
switch (N->getOpcode()) { |
1 |
| 53 |
default: |
0 |
53 |
default: |
0 |
| 54 |
#ifndef NDEBUG |
--- |
54 |
#ifndef NDEBUG |
--- |
| 55 |
dbgs() << "PromoteIntegerResult #" << ResNo << ": "; |
0 |
55 |
dbgs() << "PromoteIntegerResult #" << ResNo << ": "; |
0 |
| 56 |
N->dump(&DAG); dbgs() << "\n"; |
0 |
56 |
N->dump(&DAG); dbgs() << "\n"; |
0 |
| 57 |
#endif |
--- |
57 |
#endif |
--- |
| 58 |
report_fatal_error("Do not know how to promote this operator!"); |
0 |
58 |
report_fatal_error("Do not know how to promote this operator!"); |
0 |
| 59 |
case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break; |
0 |
59 |
case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break; |
0 |
| 60 |
case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break; |
0 |
60 |
case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break; |
0 |
| 61 |
case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break; |
0 |
61 |
case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break; |
0 |
| 62 |
case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break; |
0 |
62 |
case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break; |
0 |
| 63 |
case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break; |
0 |
63 |
case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break; |
0 |
| 64 |
case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break; |
0 |
64 |
case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break; |
0 |
| 65 |
case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break; |
0 |
65 |
case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break; |
0 |
| 66 |
case ISD::Constant: Res = PromoteIntRes_Constant(N); break; |
0 |
66 |
case ISD::Constant: Res = PromoteIntRes_Constant(N); break; |
0 |
| 67 |
case ISD::CTLZ_ZERO_UNDEF: |
0 |
67 |
case ISD::CTLZ_ZERO_UNDEF: |
0 |
| 68 |
case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break; |
0 |
68 |
case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break; |
0 |
| 69 |
case ISD::PARITY: |
0 |
69 |
case ISD::PARITY: |
0 |
| 70 |
case ISD::CTPOP: Res = PromoteIntRes_CTPOP_PARITY(N); break; |
0 |
70 |
case ISD::CTPOP: Res = PromoteIntRes_CTPOP_PARITY(N); break; |
0 |
| 71 |
case ISD::CTTZ_ZERO_UNDEF: |
0 |
71 |
case ISD::CTTZ_ZERO_UNDEF: |
0 |
| 72 |
case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break; |
0 |
72 |
case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break; |
0 |
| 73 |
case ISD::EXTRACT_VECTOR_ELT: |
0 |
73 |
case ISD::EXTRACT_VECTOR_ELT: |
0 |
| 74 |
Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break; |
0 |
74 |
Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break; |
0 |
| 75 |
case ISD::LOAD: Res = PromoteIntRes_LOAD(cast(N)); break; |
0 |
75 |
case ISD::LOAD: Res = PromoteIntRes_LOAD(cast(N)); break; |
0 |
| 76 |
case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast(N)); |
0 |
76 |
case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast(N)); |
0 |
| 77 |
break; |
0 |
77 |
break; |
0 |
| 78 |
case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast(N)); |
0 |
78 |
case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast(N)); |
0 |
| 79 |
break; |
0 |
79 |
break; |
0 |
| 80 |
case ISD::SELECT: |
0 |
80 |
case ISD::SELECT: |
0 |
| 81 |
case ISD::VSELECT: |
--- |
81 |
case ISD::VSELECT: |
--- |
| 82 |
case ISD::VP_SELECT: |
--- |
82 |
case ISD::VP_SELECT: |
--- |
| 83 |
case ISD::VP_MERGE: |
--- |
83 |
case ISD::VP_MERGE: |
--- |
| 84 |
Res = PromoteIntRes_Select(N); |
0 |
84 |
Res = PromoteIntRes_Select(N); |
0 |
| 85 |
break; |
0 |
85 |
break; |
0 |
| 86 |
case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break; |
0 |
86 |
case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break; |
0 |
| 87 |
case ISD::STRICT_FSETCC: |
1 |
87 |
case ISD::STRICT_FSETCC: |
1 |
| 88 |
case ISD::STRICT_FSETCCS: |
--- |
88 |
case ISD::STRICT_FSETCCS: |
--- |
| 89 |
case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break; |
1 |
89 |
case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break; |
1 |
| 90 |
case ISD::SMIN: |
0 |
90 |
case ISD::SMIN: |
0 |
| 91 |
case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break; |
0 |
91 |
case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break; |
0 |
| 92 |
case ISD::UMIN: |
0 |
92 |
case ISD::UMIN: |
0 |
| 93 |
case ISD::UMAX: Res = PromoteIntRes_UMINUMAX(N); break; |
0 |
93 |
case ISD::UMAX: Res = PromoteIntRes_UMINUMAX(N); break; |
0 |
| 94 |
|
--- |
94 |
|
--- |
| 95 |
case ISD::SHL: |
0 |
95 |
case ISD::SHL: |
0 |
| 96 |
case ISD::VP_SHL: Res = PromoteIntRes_SHL(N); break; |
0 |
96 |
case ISD::VP_SHL: Res = PromoteIntRes_SHL(N); break; |
0 |
| 97 |
case ISD::SIGN_EXTEND_INREG: |
0 |
97 |
case ISD::SIGN_EXTEND_INREG: |
0 |
| 98 |
Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break; |
0 |
98 |
Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break; |
0 |
| 99 |
case ISD::SRA: |
0 |
99 |
case ISD::SRA: |
0 |
| 100 |
case ISD::VP_ASHR: Res = PromoteIntRes_SRA(N); break; |
0 |
100 |
case ISD::VP_ASHR: Res = PromoteIntRes_SRA(N); break; |
0 |
| 101 |
case ISD::SRL: |
0 |
101 |
case ISD::SRL: |
0 |
| 102 |
case ISD::VP_LSHR: Res = PromoteIntRes_SRL(N); break; |
0 |
102 |
case ISD::VP_LSHR: Res = PromoteIntRes_SRL(N); break; |
0 |
| 103 |
case ISD::VP_TRUNCATE: |
0 |
103 |
case ISD::VP_TRUNCATE: |
0 |
| 104 |
case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break; |
0 |
104 |
case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break; |
0 |
| 105 |
case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break; |
0 |
105 |
case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break; |
0 |
| 106 |
case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break; |
0 |
106 |
case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break; |
0 |
| 107 |
case ISD::VSCALE: Res = PromoteIntRes_VSCALE(N); break; |
0 |
107 |
case ISD::VSCALE: Res = PromoteIntRes_VSCALE(N); break; |
0 |
| 108 |
|
--- |
108 |
|
--- |
| 109 |
case ISD::EXTRACT_SUBVECTOR: |
0 |
109 |
case ISD::EXTRACT_SUBVECTOR: |
0 |
| 110 |
Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break; |
0 |
110 |
Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break; |
0 |
| 111 |
case ISD::INSERT_SUBVECTOR: |
0 |
111 |
case ISD::INSERT_SUBVECTOR: |
0 |
| 112 |
Res = PromoteIntRes_INSERT_SUBVECTOR(N); break; |
0 |
112 |
Res = PromoteIntRes_INSERT_SUBVECTOR(N); break; |
0 |
| 113 |
case ISD::VECTOR_REVERSE: |
0 |
113 |
case ISD::VECTOR_REVERSE: |
0 |
| 114 |
Res = PromoteIntRes_VECTOR_REVERSE(N); break; |
0 |
114 |
Res = PromoteIntRes_VECTOR_REVERSE(N); break; |
0 |
| 115 |
case ISD::VECTOR_SHUFFLE: |
0 |
115 |
case ISD::VECTOR_SHUFFLE: |
0 |
| 116 |
Res = PromoteIntRes_VECTOR_SHUFFLE(N); break; |
0 |
116 |
Res = PromoteIntRes_VECTOR_SHUFFLE(N); break; |
0 |
| 117 |
case ISD::VECTOR_SPLICE: |
0 |
117 |
case ISD::VECTOR_SPLICE: |
0 |
| 118 |
Res = PromoteIntRes_VECTOR_SPLICE(N); break; |
0 |
118 |
Res = PromoteIntRes_VECTOR_SPLICE(N); break; |
0 |
| 119 |
case ISD::VECTOR_INTERLEAVE: |
0 |
119 |
case ISD::VECTOR_INTERLEAVE: |
0 |
| 120 |
case ISD::VECTOR_DEINTERLEAVE: |
--- |
120 |
case ISD::VECTOR_DEINTERLEAVE: |
--- |
| 121 |
Res = PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(N); |
0 |
121 |
Res = PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(N); |
0 |
| 122 |
return; |
0 |
122 |
return; |
0 |
| 123 |
case ISD::INSERT_VECTOR_ELT: |
0 |
123 |
case ISD::INSERT_VECTOR_ELT: |
0 |
| 124 |
Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break; |
0 |
124 |
Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break; |
0 |
| 125 |
case ISD::BUILD_VECTOR: |
0 |
125 |
case ISD::BUILD_VECTOR: |
0 |
| 126 |
Res = PromoteIntRes_BUILD_VECTOR(N); |
0 |
126 |
Res = PromoteIntRes_BUILD_VECTOR(N); |
0 |
| 127 |
break; |
0 |
127 |
break; |
0 |
| 128 |
case ISD::SPLAT_VECTOR: |
0 |
128 |
case ISD::SPLAT_VECTOR: |
0 |
| 129 |
case ISD::SCALAR_TO_VECTOR: |
--- |
129 |
case ISD::SCALAR_TO_VECTOR: |
--- |
| 130 |
Res = PromoteIntRes_ScalarOp(N); |
0 |
130 |
Res = PromoteIntRes_ScalarOp(N); |
0 |
| 131 |
break; |
0 |
131 |
break; |
0 |
| 132 |
case ISD::STEP_VECTOR: Res = PromoteIntRes_STEP_VECTOR(N); break; |
0 |
132 |
case ISD::STEP_VECTOR: Res = PromoteIntRes_STEP_VECTOR(N); break; |
0 |
| 133 |
case ISD::CONCAT_VECTORS: |
0 |
133 |
case ISD::CONCAT_VECTORS: |
0 |
| 134 |
Res = PromoteIntRes_CONCAT_VECTORS(N); break; |
0 |
134 |
Res = PromoteIntRes_CONCAT_VECTORS(N); break; |
0 |
| 135 |
|
--- |
135 |
|
--- |
| 136 |
case ISD::ANY_EXTEND_VECTOR_INREG: |
0 |
136 |
case ISD::ANY_EXTEND_VECTOR_INREG: |
0 |
| 137 |
case ISD::SIGN_EXTEND_VECTOR_INREG: |
--- |
137 |
case ISD::SIGN_EXTEND_VECTOR_INREG: |
--- |
| 138 |
case ISD::ZERO_EXTEND_VECTOR_INREG: |
--- |
138 |
case ISD::ZERO_EXTEND_VECTOR_INREG: |
--- |
| 139 |
Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break; |
0 |
139 |
Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break; |
0 |
| 140 |
|
--- |
140 |
|
--- |
| 141 |
case ISD::SIGN_EXTEND: |
0 |
141 |
case ISD::SIGN_EXTEND: |
0 |
| 142 |
case ISD::VP_SIGN_EXTEND: |
--- |
142 |
case ISD::VP_SIGN_EXTEND: |
--- |
| 143 |
case ISD::ZERO_EXTEND: |
--- |
143 |
case ISD::ZERO_EXTEND: |
--- |
| 144 |
case ISD::VP_ZERO_EXTEND: |
--- |
144 |
case ISD::VP_ZERO_EXTEND: |
--- |
| 145 |
case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break; |
0 |
145 |
case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break; |
0 |
| 146 |
|
--- |
146 |
|
--- |
| 147 |
case ISD::VP_FP_TO_SINT: |
0 |
147 |
case ISD::VP_FP_TO_SINT: |
0 |
| 148 |
case ISD::VP_FP_TO_UINT: |
--- |
148 |
case ISD::VP_FP_TO_UINT: |
--- |
| 149 |
case ISD::STRICT_FP_TO_SINT: |
--- |
149 |
case ISD::STRICT_FP_TO_SINT: |
--- |
| 150 |
case ISD::STRICT_FP_TO_UINT: |
--- |
150 |
case ISD::STRICT_FP_TO_UINT: |
--- |
| 151 |
case ISD::FP_TO_SINT: |
--- |
151 |
case ISD::FP_TO_SINT: |
--- |
| 152 |
case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break; |
0 |
152 |
case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break; |
0 |
| 153 |
|
--- |
153 |
|
--- |
| 154 |
case ISD::FP_TO_SINT_SAT: |
0 |
154 |
case ISD::FP_TO_SINT_SAT: |
0 |
| 155 |
case ISD::FP_TO_UINT_SAT: |
--- |
155 |
case ISD::FP_TO_UINT_SAT: |
--- |
| 156 |
Res = PromoteIntRes_FP_TO_XINT_SAT(N); break; |
0 |
156 |
Res = PromoteIntRes_FP_TO_XINT_SAT(N); break; |
0 |
| 157 |
|
--- |
157 |
|
--- |
| 158 |
case ISD::FP_TO_BF16: |
0 |
158 |
case ISD::FP_TO_BF16: |
0 |
| 159 |
case ISD::FP_TO_FP16: |
--- |
159 |
case ISD::FP_TO_FP16: |
--- |
| 160 |
Res = PromoteIntRes_FP_TO_FP16_BF16(N); |
0 |
160 |
Res = PromoteIntRes_FP_TO_FP16_BF16(N); |
0 |
| 161 |
break; |
0 |
161 |
break; |
0 |
| 162 |
|
--- |
162 |
|
--- |
| 163 |
case ISD::GET_ROUNDING: Res = PromoteIntRes_GET_ROUNDING(N); break; |
0 |
163 |
case ISD::GET_ROUNDING: Res = PromoteIntRes_GET_ROUNDING(N); break; |
0 |
| 164 |
|
--- |
164 |
|
--- |
| 165 |
case ISD::AND: |
0 |
165 |
case ISD::AND: |
0 |
| 166 |
case ISD::OR: |
--- |
166 |
case ISD::OR: |
--- |
| 167 |
case ISD::XOR: |
--- |
167 |
case ISD::XOR: |
--- |
| 168 |
case ISD::ADD: |
--- |
168 |
case ISD::ADD: |
--- |
| 169 |
case ISD::SUB: |
--- |
169 |
case ISD::SUB: |
--- |
| 170 |
case ISD::MUL: |
--- |
170 |
case ISD::MUL: |
--- |
| 171 |
case ISD::VP_AND: |
--- |
171 |
case ISD::VP_AND: |
--- |
| 172 |
case ISD::VP_OR: |
--- |
172 |
case ISD::VP_OR: |
--- |
| 173 |
case ISD::VP_XOR: |
--- |
173 |
case ISD::VP_XOR: |
--- |
| 174 |
case ISD::VP_ADD: |
--- |
174 |
case ISD::VP_ADD: |
--- |
| 175 |
case ISD::VP_SUB: |
--- |
175 |
case ISD::VP_SUB: |
--- |
| 176 |
case ISD::VP_MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break; |
0 |
176 |
case ISD::VP_MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break; |
0 |
| 177 |
|
--- |
177 |
|
--- |
| 178 |
case ISD::VP_SMIN: |
0 |
178 |
case ISD::VP_SMIN: |
0 |
| 179 |
case ISD::VP_SMAX: |
--- |
179 |
case ISD::VP_SMAX: |
--- |
| 180 |
case ISD::SDIV: |
--- |
180 |
case ISD::SDIV: |
--- |
| 181 |
case ISD::SREM: |
--- |
181 |
case ISD::SREM: |
--- |
| 182 |
case ISD::VP_SDIV: |
--- |
182 |
case ISD::VP_SDIV: |
--- |
| 183 |
case ISD::VP_SREM: Res = PromoteIntRes_SExtIntBinOp(N); break; |
0 |
183 |
case ISD::VP_SREM: Res = PromoteIntRes_SExtIntBinOp(N); break; |
0 |
| 184 |
|
--- |
184 |
|
--- |
| 185 |
case ISD::VP_UMIN: |
0 |
185 |
case ISD::VP_UMIN: |
0 |
| 186 |
case ISD::VP_UMAX: |
--- |
186 |
case ISD::VP_UMAX: |
--- |
| 187 |
case ISD::UDIV: |
--- |
187 |
case ISD::UDIV: |
--- |
| 188 |
case ISD::UREM: |
--- |
188 |
case ISD::UREM: |
--- |
| 189 |
case ISD::VP_UDIV: |
--- |
189 |
case ISD::VP_UDIV: |
--- |
| 190 |
case ISD::VP_UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break; |
0 |
190 |
case ISD::VP_UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break; |
0 |
| 191 |
|
--- |
191 |
|
--- |
| 192 |
case ISD::SADDO: |
0 |
192 |
case ISD::SADDO: |
0 |
| 193 |
case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break; |
0 |
193 |
case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break; |
0 |
| 194 |
case ISD::UADDO: |
0 |
194 |
case ISD::UADDO: |
0 |
| 195 |
case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break; |
0 |
195 |
case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break; |
0 |
| 196 |
case ISD::SMULO: |
0 |
196 |
case ISD::SMULO: |
0 |
| 197 |
case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break; |
0 |
197 |
case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break; |
0 |
| 198 |
|
--- |
198 |
|
--- |
| 199 |
case ISD::ADDE: |
0 |
199 |
case ISD::ADDE: |
0 |
| 200 |
case ISD::SUBE: |
--- |
200 |
case ISD::SUBE: |
--- |
| 201 |
case ISD::UADDO_CARRY: |
--- |
201 |
case ISD::UADDO_CARRY: |
--- |
| 202 |
case ISD::USUBO_CARRY: Res = PromoteIntRes_UADDSUBO_CARRY(N, ResNo); break; |
0 |
202 |
case ISD::USUBO_CARRY: Res = PromoteIntRes_UADDSUBO_CARRY(N, ResNo); break; |
0 |
| 203 |
|
--- |
203 |
|
--- |
| 204 |
case ISD::SADDO_CARRY: |
0 |
204 |
case ISD::SADDO_CARRY: |
0 |
| 205 |
case ISD::SSUBO_CARRY: Res = PromoteIntRes_SADDSUBO_CARRY(N, ResNo); break; |
0 |
205 |
case ISD::SSUBO_CARRY: Res = PromoteIntRes_SADDSUBO_CARRY(N, ResNo); break; |
0 |
| 206 |
|
--- |
206 |
|
--- |
| 207 |
case ISD::SADDSAT: |
0 |
207 |
case ISD::SADDSAT: |
0 |
| 208 |
case ISD::UADDSAT: |
--- |
208 |
case ISD::UADDSAT: |
--- |
| 209 |
case ISD::SSUBSAT: |
--- |
209 |
case ISD::SSUBSAT: |
--- |
| 210 |
case ISD::USUBSAT: |
--- |
210 |
case ISD::USUBSAT: |
--- |
| 211 |
case ISD::SSHLSAT: |
--- |
211 |
case ISD::SSHLSAT: |
--- |
| 212 |
case ISD::USHLSAT: Res = PromoteIntRes_ADDSUBSHLSAT(N); break; |
0 |
212 |
case ISD::USHLSAT: Res = PromoteIntRes_ADDSUBSHLSAT(N); break; |
0 |
| 213 |
|
--- |
213 |
|
--- |
| 214 |
case ISD::SMULFIX: |
0 |
214 |
case ISD::SMULFIX: |
0 |
| 215 |
case ISD::SMULFIXSAT: |
--- |
215 |
case ISD::SMULFIXSAT: |
--- |
| 216 |
case ISD::UMULFIX: |
--- |
216 |
case ISD::UMULFIX: |
--- |
| 217 |
case ISD::UMULFIXSAT: Res = PromoteIntRes_MULFIX(N); break; |
0 |
217 |
case ISD::UMULFIXSAT: Res = PromoteIntRes_MULFIX(N); break; |
0 |
| 218 |
|
--- |
218 |
|
--- |
| 219 |
case ISD::SDIVFIX: |
0 |
219 |
case ISD::SDIVFIX: |
0 |
| 220 |
case ISD::SDIVFIXSAT: |
--- |
220 |
case ISD::SDIVFIXSAT: |
--- |
| 221 |
case ISD::UDIVFIX: |
--- |
221 |
case ISD::UDIVFIX: |
--- |
| 222 |
case ISD::UDIVFIXSAT: Res = PromoteIntRes_DIVFIX(N); break; |
0 |
222 |
case ISD::UDIVFIXSAT: Res = PromoteIntRes_DIVFIX(N); break; |
0 |
| 223 |
|
--- |
223 |
|
--- |
| 224 |
case ISD::ABS: Res = PromoteIntRes_ABS(N); break; |
0 |
224 |
case ISD::ABS: Res = PromoteIntRes_ABS(N); break; |
0 |
| 225 |
|
--- |
225 |
|
--- |
| 226 |
case ISD::ATOMIC_LOAD: |
0 |
226 |
case ISD::ATOMIC_LOAD: |
0 |
| 227 |
Res = PromoteIntRes_Atomic0(cast(N)); break; |
0 |
227 |
Res = PromoteIntRes_Atomic0(cast(N)); break; |
0 |
| 228 |
|
--- |
228 |
|
--- |
| 229 |
case ISD::ATOMIC_LOAD_ADD: |
0 |
229 |
case ISD::ATOMIC_LOAD_ADD: |
0 |
| 230 |
case ISD::ATOMIC_LOAD_SUB: |
--- |
230 |
case ISD::ATOMIC_LOAD_SUB: |
--- |
| 231 |
case ISD::ATOMIC_LOAD_AND: |
--- |
231 |
case ISD::ATOMIC_LOAD_AND: |
--- |
| 232 |
case ISD::ATOMIC_LOAD_CLR: |
--- |
232 |
case ISD::ATOMIC_LOAD_CLR: |
--- |
| 233 |
case ISD::ATOMIC_LOAD_OR: |
--- |
233 |
case ISD::ATOMIC_LOAD_OR: |
--- |
| 234 |
case ISD::ATOMIC_LOAD_XOR: |
--- |
234 |
case ISD::ATOMIC_LOAD_XOR: |
--- |
| 235 |
case ISD::ATOMIC_LOAD_NAND: |
--- |
235 |
case ISD::ATOMIC_LOAD_NAND: |
--- |
| 236 |
case ISD::ATOMIC_LOAD_MIN: |
--- |
236 |
case ISD::ATOMIC_LOAD_MIN: |
--- |
| 237 |
case ISD::ATOMIC_LOAD_MAX: |
--- |
237 |
case ISD::ATOMIC_LOAD_MAX: |
--- |
| 238 |
case ISD::ATOMIC_LOAD_UMIN: |
--- |
238 |
case ISD::ATOMIC_LOAD_UMIN: |
--- |
| 239 |
case ISD::ATOMIC_LOAD_UMAX: |
--- |
239 |
case ISD::ATOMIC_LOAD_UMAX: |
--- |
| 240 |
case ISD::ATOMIC_SWAP: |
--- |
240 |
case ISD::ATOMIC_SWAP: |
--- |
| 241 |
Res = PromoteIntRes_Atomic1(cast(N)); break; |
0 |
241 |
Res = PromoteIntRes_Atomic1(cast(N)); break; |
0 |
| 242 |
|
--- |
242 |
|
--- |
| 243 |
case ISD::ATOMIC_CMP_SWAP: |
0 |
243 |
case ISD::ATOMIC_CMP_SWAP: |
0 |
| 244 |
case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: |
--- |
244 |
case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: |
--- |
| 245 |
Res = PromoteIntRes_AtomicCmpSwap(cast(N), ResNo); |
0 |
245 |
Res = PromoteIntRes_AtomicCmpSwap(cast(N), ResNo); |
0 |
| 246 |
break; |
0 |
246 |
break; |
0 |
| 247 |
|
--- |
247 |
|
--- |
| 248 |
case ISD::VECREDUCE_ADD: |
0 |
248 |
case ISD::VECREDUCE_ADD: |
0 |
| 249 |
case ISD::VECREDUCE_MUL: |
--- |
249 |
case ISD::VECREDUCE_MUL: |
--- |
| 250 |
case ISD::VECREDUCE_AND: |
--- |
250 |
case ISD::VECREDUCE_AND: |
--- |
| 251 |
case ISD::VECREDUCE_OR: |
--- |
251 |
case ISD::VECREDUCE_OR: |
--- |
| 252 |
case ISD::VECREDUCE_XOR: |
--- |
252 |
case ISD::VECREDUCE_XOR: |
--- |
| 253 |
case ISD::VECREDUCE_SMAX: |
--- |
253 |
case ISD::VECREDUCE_SMAX: |
--- |
| 254 |
case ISD::VECREDUCE_SMIN: |
--- |
254 |
case ISD::VECREDUCE_SMIN: |
--- |
| 255 |
case ISD::VECREDUCE_UMAX: |
--- |
255 |
case ISD::VECREDUCE_UMAX: |
--- |
| 256 |
case ISD::VECREDUCE_UMIN: |
--- |
256 |
case ISD::VECREDUCE_UMIN: |
--- |
| 257 |
Res = PromoteIntRes_VECREDUCE(N); |
0 |
257 |
Res = PromoteIntRes_VECREDUCE(N); |
0 |
| 258 |
break; |
0 |
258 |
break; |
0 |
| 259 |
|
--- |
259 |
|
--- |
| 260 |
case ISD::VP_REDUCE_ADD: |
0 |
260 |
case ISD::VP_REDUCE_ADD: |
0 |
| 261 |
case ISD::VP_REDUCE_MUL: |
--- |
261 |
case ISD::VP_REDUCE_MUL: |
--- |
| 262 |
case ISD::VP_REDUCE_AND: |
--- |
262 |
case ISD::VP_REDUCE_AND: |
--- |
| 263 |
case ISD::VP_REDUCE_OR: |
--- |
263 |
case ISD::VP_REDUCE_OR: |
--- |
| 264 |
case ISD::VP_REDUCE_XOR: |
--- |
264 |
case ISD::VP_REDUCE_XOR: |
--- |
| 265 |
case ISD::VP_REDUCE_SMAX: |
--- |
265 |
case ISD::VP_REDUCE_SMAX: |
--- |
| 266 |
case ISD::VP_REDUCE_SMIN: |
--- |
266 |
case ISD::VP_REDUCE_SMIN: |
--- |
| 267 |
case ISD::VP_REDUCE_UMAX: |
--- |
267 |
case ISD::VP_REDUCE_UMAX: |
--- |
| 268 |
case ISD::VP_REDUCE_UMIN: |
--- |
268 |
case ISD::VP_REDUCE_UMIN: |
--- |
| 269 |
Res = PromoteIntRes_VP_REDUCE(N); |
0 |
269 |
Res = PromoteIntRes_VP_REDUCE(N); |
0 |
| 270 |
break; |
0 |
270 |
break; |
0 |
| 271 |
|
--- |
271 |
|
--- |
| 272 |
case ISD::FREEZE: |
0 |
272 |
case ISD::FREEZE: |
0 |
| 273 |
Res = PromoteIntRes_FREEZE(N); |
0 |
273 |
Res = PromoteIntRes_FREEZE(N); |
0 |
| 274 |
break; |
0 |
274 |
break; |
0 |
| 275 |
|
--- |
275 |
|
--- |
| 276 |
case ISD::ROTL: |
0 |
276 |
case ISD::ROTL: |
0 |
| 277 |
case ISD::ROTR: |
--- |
277 |
case ISD::ROTR: |
--- |
| 278 |
Res = PromoteIntRes_Rotate(N); |
0 |
278 |
Res = PromoteIntRes_Rotate(N); |
0 |
| 279 |
break; |
0 |
279 |
break; |
0 |
| 280 |
|
--- |
280 |
|
--- |
| 281 |
case ISD::FSHL: |
0 |
281 |
case ISD::FSHL: |
0 |
| 282 |
case ISD::FSHR: |
--- |
282 |
case ISD::FSHR: |
--- |
| 283 |
Res = PromoteIntRes_FunnelShift(N); |
0 |
283 |
Res = PromoteIntRes_FunnelShift(N); |
0 |
| 284 |
break; |
0 |
284 |
break; |
0 |
| 285 |
|
--- |
285 |
|
--- |
| 286 |
case ISD::IS_FPCLASS: |
0 |
286 |
case ISD::IS_FPCLASS: |
0 |
| 287 |
Res = PromoteIntRes_IS_FPCLASS(N); |
0 |
287 |
Res = PromoteIntRes_IS_FPCLASS(N); |
0 |
| 288 |
break; |
0 |
288 |
break; |
0 |
| 289 |
case ISD::FFREXP: |
0 |
289 |
case ISD::FFREXP: |
0 |
| 290 |
Res = PromoteIntRes_FFREXP(N); |
0 |
290 |
Res = PromoteIntRes_FFREXP(N); |
0 |
| 291 |
break; |
0 |
291 |
break; |
0 |
| 292 |
} |
--- |
292 |
} |
--- |
| 293 |
|
--- |
293 |
|
--- |
| 294 |
// If the result is null then the sub-method took care of registering it. |
--- |
294 |
// If the result is null then the sub-method took care of registering it. |
--- |
| 295 |
if (Res.getNode()) |
1 |
295 |
if (Res.getNode()) |
1 |
| 296 |
SetPromotedInteger(SDValue(N, ResNo), Res); |
1 |
296 |
SetPromotedInteger(SDValue(N, ResNo), Res); |
1 |
| 297 |
} |
--- |
297 |
} |
--- |
| 298 |
|
--- |
298 |
|
--- |
| 299 |
SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N, |
0 |
299 |
SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N, |
0 |
| 300 |
unsigned ResNo) { |
--- |
300 |
unsigned ResNo) { |
--- |
| 301 |
SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); |
0 |
301 |
SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); |
0 |
| 302 |
return GetPromotedInteger(Op); |
0 |
302 |
return GetPromotedInteger(Op); |
0 |
| 303 |
} |
--- |
303 |
} |
--- |
| 304 |
|
--- |
304 |
|
--- |
| 305 |
SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) { |
0 |
305 |
SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) { |
0 |
| 306 |
// Sign-extend the new bits, and continue the assertion. |
--- |
306 |
// Sign-extend the new bits, and continue the assertion. |
--- |
| 307 |
SDValue Op = SExtPromotedInteger(N->getOperand(0)); |
0 |
307 |
SDValue Op = SExtPromotedInteger(N->getOperand(0)); |
0 |
| 308 |
return DAG.getNode(ISD::AssertSext, SDLoc(N), |
0 |
308 |
return DAG.getNode(ISD::AssertSext, SDLoc(N), |
0 |
| 309 |
Op.getValueType(), Op, N->getOperand(1)); |
0 |
309 |
Op.getValueType(), Op, N->getOperand(1)); |
0 |
| 310 |
} |
--- |
310 |
} |
--- |
| 311 |
|
--- |
311 |
|
--- |
| 312 |
SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) { |
0 |
312 |
SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) { |
0 |
| 313 |
// Zero the new bits, and continue the assertion. |
--- |
313 |
// Zero the new bits, and continue the assertion. |
--- |
| 314 |
SDValue Op = ZExtPromotedInteger(N->getOperand(0)); |
0 |
314 |
SDValue Op = ZExtPromotedInteger(N->getOperand(0)); |
0 |
| 315 |
return DAG.getNode(ISD::AssertZext, SDLoc(N), |
0 |
315 |
return DAG.getNode(ISD::AssertZext, SDLoc(N), |
0 |
| 316 |
Op.getValueType(), Op, N->getOperand(1)); |
0 |
316 |
Op.getValueType(), Op, N->getOperand(1)); |
0 |
| 317 |
} |
--- |
317 |
} |
--- |
| 318 |
|
--- |
318 |
|
--- |
| 319 |
SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) { |
0 |
319 |
SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) { |
0 |
| 320 |
EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
320 |
EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
| 321 |
SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N), |
0 |
321 |
SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N), |
0 |
| 322 |
N->getMemoryVT(), ResVT, |
--- |
322 |
N->getMemoryVT(), ResVT, |
--- |
| 323 |
N->getChain(), N->getBasePtr(), |
0 |
323 |
N->getChain(), N->getBasePtr(), |
0 |
| 324 |
N->getMemOperand()); |
--- |
324 |
N->getMemOperand()); |
--- |
| 325 |
// Legalize the chain result - switch anything that used the old chain to |
--- |
325 |
// Legalize the chain result - switch anything that used the old chain to |
--- |
| 326 |
// use the new one. |
--- |
326 |
// use the new one. |
--- |
| 327 |
ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); |
0 |
327 |
ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); |
0 |
| 328 |
return Res; |
0 |
328 |
return Res; |
0 |
| 329 |
} |
--- |
329 |
} |
--- |
| 330 |
|
--- |
330 |
|
--- |
| 331 |
SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) { |
0 |
331 |
SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) { |
0 |
| 332 |
SDValue Op2 = GetPromotedInteger(N->getOperand(2)); |
0 |
332 |
SDValue Op2 = GetPromotedInteger(N->getOperand(2)); |
0 |
| 333 |
SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N), |
0 |
333 |
SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N), |
0 |
| 334 |
N->getMemoryVT(), |
--- |
334 |
N->getMemoryVT(), |
--- |
| 335 |
N->getChain(), N->getBasePtr(), |
0 |
335 |
N->getChain(), N->getBasePtr(), |
0 |
| 336 |
Op2, N->getMemOperand()); |
--- |
336 |
Op2, N->getMemOperand()); |
--- |
| 337 |
// Legalize the chain result - switch anything that used the old chain to |
--- |
337 |
// Legalize the chain result - switch anything that used the old chain to |
--- |
| 338 |
// use the new one. |
--- |
338 |
// use the new one. |
--- |
| 339 |
ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); |
0 |
339 |
ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); |
0 |
| 340 |
return Res; |
0 |
340 |
return Res; |
0 |
| 341 |
} |
--- |
341 |
} |
--- |
| 342 |
|
--- |
342 |
|
--- |
| 343 |
SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N, |
0 |
343 |
SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N, |
0 |
| 344 |
unsigned ResNo) { |
--- |
344 |
unsigned ResNo) { |
--- |
| 345 |
if (ResNo == 1) { |
0 |
345 |
if (ResNo == 1) { |
0 |
| 346 |
assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); |
0 |
346 |
assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); |
0 |
| 347 |
EVT SVT = getSetCCResultType(N->getOperand(2).getValueType()); |
0 |
347 |
EVT SVT = getSetCCResultType(N->getOperand(2).getValueType()); |
0 |
| 348 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); |
0 |
348 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); |
0 |
| 349 |
|
--- |
349 |
|
--- |
| 350 |
// Only use the result of getSetCCResultType if it is legal, |
--- |
350 |
// Only use the result of getSetCCResultType if it is legal, |
--- |
| 351 |
// otherwise just use the promoted result type (NVT). |
--- |
351 |
// otherwise just use the promoted result type (NVT). |
--- |
| 352 |
if (!TLI.isTypeLegal(SVT)) |
0 |
352 |
if (!TLI.isTypeLegal(SVT)) |
0 |
| 353 |
SVT = NVT; |
0 |
353 |
SVT = NVT; |
0 |
| 354 |
|
--- |
354 |
|
--- |
| 355 |
SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other); |
0 |
355 |
SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other); |
0 |
| 356 |
SDValue Res = DAG.getAtomicCmpSwap( |
0 |
356 |
SDValue Res = DAG.getAtomicCmpSwap( |
0 |
| 357 |
ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs, |
0 |
357 |
ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs, |
0 |
| 358 |
N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3), |
0 |
358 |
N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3), |
0 |
| 359 |
N->getMemOperand()); |
--- |
359 |
N->getMemOperand()); |
--- |
| 360 |
ReplaceValueWith(SDValue(N, 0), Res.getValue(0)); |
0 |
360 |
ReplaceValueWith(SDValue(N, 0), Res.getValue(0)); |
0 |
| 361 |
ReplaceValueWith(SDValue(N, 2), Res.getValue(2)); |
0 |
361 |
ReplaceValueWith(SDValue(N, 2), Res.getValue(2)); |
0 |
| 362 |
return Res.getValue(1); |
0 |
362 |
return Res.getValue(1); |
0 |
| 363 |
} |
--- |
363 |
} |
--- |
| 364 |
|
--- |
364 |
|
--- |
| 365 |
// Op2 is used for the comparison and thus must be extended according to the |
--- |
365 |
// Op2 is used for the comparison and thus must be extended according to the |
--- |
| 366 |
// target's atomic operations. Op3 is merely stored and so can be left alone. |
--- |
366 |
// target's atomic operations. Op3 is merely stored and so can be left alone. |
--- |
| 367 |
SDValue Op2 = N->getOperand(2); |
0 |
367 |
SDValue Op2 = N->getOperand(2); |
0 |
| 368 |
SDValue Op3 = GetPromotedInteger(N->getOperand(3)); |
0 |
368 |
SDValue Op3 = GetPromotedInteger(N->getOperand(3)); |
0 |
| 369 |
switch (TLI.getExtendForAtomicCmpSwapArg()) { |
0 |
369 |
switch (TLI.getExtendForAtomicCmpSwapArg()) { |
0 |
| 370 |
case ISD::SIGN_EXTEND: |
0 |
370 |
case ISD::SIGN_EXTEND: |
0 |
| 371 |
Op2 = SExtPromotedInteger(Op2); |
0 |
371 |
Op2 = SExtPromotedInteger(Op2); |
0 |
| 372 |
break; |
0 |
372 |
break; |
0 |
| 373 |
case ISD::ZERO_EXTEND: |
0 |
373 |
case ISD::ZERO_EXTEND: |
0 |
| 374 |
Op2 = ZExtPromotedInteger(Op2); |
0 |
374 |
Op2 = ZExtPromotedInteger(Op2); |
0 |
| 375 |
break; |
0 |
375 |
break; |
0 |
| 376 |
case ISD::ANY_EXTEND: |
0 |
376 |
case ISD::ANY_EXTEND: |
0 |
| 377 |
Op2 = GetPromotedInteger(Op2); |
0 |
377 |
Op2 = GetPromotedInteger(Op2); |
0 |
| 378 |
break; |
0 |
378 |
break; |
0 |
| 379 |
default: |
0 |
379 |
default: |
0 |
| 380 |
llvm_unreachable("Invalid atomic op extension"); |
0 |
380 |
llvm_unreachable("Invalid atomic op extension"); |
0 |
| 381 |
} |
--- |
381 |
} |
--- |
| 382 |
|
--- |
382 |
|
--- |
| 383 |
SDVTList VTs = |
--- |
383 |
SDVTList VTs = |
--- |
| 384 |
DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other); |
0 |
384 |
DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other); |
0 |
| 385 |
SDValue Res = DAG.getAtomicCmpSwap( |
0 |
385 |
SDValue Res = DAG.getAtomicCmpSwap( |
0 |
| 386 |
N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(), |
0 |
386 |
N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(), |
0 |
| 387 |
N->getBasePtr(), Op2, Op3, N->getMemOperand()); |
0 |
387 |
N->getBasePtr(), Op2, Op3, N->getMemOperand()); |
0 |
| 388 |
// Update the use to N with the newly created Res. |
--- |
388 |
// Update the use to N with the newly created Res. |
--- |
| 389 |
for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i) |
0 |
389 |
for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i) |
0 |
| 390 |
ReplaceValueWith(SDValue(N, i), Res.getValue(i)); |
0 |
390 |
ReplaceValueWith(SDValue(N, i), Res.getValue(i)); |
0 |
| 391 |
return Res; |
0 |
391 |
return Res; |
0 |
| 392 |
} |
--- |
392 |
} |
--- |
| 393 |
|
--- |
393 |
|
--- |
| 394 |
SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) { |
0 |
394 |
SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) { |
0 |
| 395 |
SDValue InOp = N->getOperand(0); |
0 |
395 |
SDValue InOp = N->getOperand(0); |
0 |
| 396 |
EVT InVT = InOp.getValueType(); |
0 |
396 |
EVT InVT = InOp.getValueType(); |
0 |
| 397 |
EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT); |
0 |
397 |
EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT); |
0 |
| 398 |
EVT OutVT = N->getValueType(0); |
0 |
398 |
EVT OutVT = N->getValueType(0); |
0 |
| 399 |
EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); |
0 |
399 |
EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); |
0 |
| 400 |
SDLoc dl(N); |
0 |
400 |
SDLoc dl(N); |
0 |
| 401 |
|
--- |
401 |
|
--- |
| 402 |
switch (getTypeAction(InVT)) { |
0 |
402 |
switch (getTypeAction(InVT)) { |
0 |
| 403 |
case TargetLowering::TypeLegal: |
0 |
403 |
case TargetLowering::TypeLegal: |
0 |
| 404 |
break; |
0 |
404 |
break; |
0 |
| 405 |
case TargetLowering::TypePromoteInteger: |
0 |
405 |
case TargetLowering::TypePromoteInteger: |
0 |
| 406 |
if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector()) |
0 |
406 |
if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector()) |
0 |
| 407 |
// The input promotes to the same size. Convert the promoted value. |
--- |
407 |
// The input promotes to the same size. Convert the promoted value. |
--- |
| 408 |
return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp)); |
0 |
408 |
return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp)); |
0 |
| 409 |
break; |
0 |
409 |
break; |
0 |
| 410 |
case TargetLowering::TypeSoftenFloat: |
0 |
410 |
case TargetLowering::TypeSoftenFloat: |
0 |
| 411 |
// Promote the integer operand by hand. |
--- |
411 |
// Promote the integer operand by hand. |
--- |
| 412 |
return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp)); |
0 |
412 |
return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp)); |
0 |
| 413 |
case TargetLowering::TypeSoftPromoteHalf: |
0 |
413 |
case TargetLowering::TypeSoftPromoteHalf: |
0 |
| 414 |
// Promote the integer operand by hand. |
--- |
414 |
// Promote the integer operand by hand. |
--- |
| 415 |
return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftPromotedHalf(InOp)); |
0 |
415 |
return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftPromotedHalf(InOp)); |
0 |
| 416 |
case TargetLowering::TypePromoteFloat: { |
0 |
416 |
case TargetLowering::TypePromoteFloat: { |
0 |
| 417 |
// Convert the promoted float by hand. |
--- |
417 |
// Convert the promoted float by hand. |
--- |
| 418 |
if (!NOutVT.isVector()) |
0 |
418 |
if (!NOutVT.isVector()) |
0 |
| 419 |
return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp)); |
0 |
419 |
return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp)); |
0 |
| 420 |
break; |
0 |
420 |
break; |
0 |
| 421 |
} |
--- |
421 |
} |
--- |
| 422 |
case TargetLowering::TypeExpandInteger: |
0 |
422 |
case TargetLowering::TypeExpandInteger: |
0 |
| 423 |
case TargetLowering::TypeExpandFloat: |
--- |
423 |
case TargetLowering::TypeExpandFloat: |
--- |
| 424 |
break; |
0 |
424 |
break; |
0 |
| 425 |
case TargetLowering::TypeScalarizeVector: |
0 |
425 |
case TargetLowering::TypeScalarizeVector: |
0 |
| 426 |
// Convert the element to an integer and promote it by hand. |
--- |
426 |
// Convert the element to an integer and promote it by hand. |
--- |
| 427 |
if (!NOutVT.isVector()) |
0 |
427 |
if (!NOutVT.isVector()) |
0 |
| 428 |
return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, |
0 |
428 |
return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, |
0 |
| 429 |
BitConvertToInteger(GetScalarizedVector(InOp))); |
0 |
429 |
BitConvertToInteger(GetScalarizedVector(InOp))); |
0 |
| 430 |
break; |
0 |
430 |
break; |
0 |
| 431 |
case TargetLowering::TypeScalarizeScalableVector: |
0 |
431 |
case TargetLowering::TypeScalarizeScalableVector: |
0 |
| 432 |
report_fatal_error("Scalarization of scalable vectors is not supported."); |
0 |
432 |
report_fatal_error("Scalarization of scalable vectors is not supported."); |
0 |
| 433 |
case TargetLowering::TypeSplitVector: { |
0 |
433 |
case TargetLowering::TypeSplitVector: { |
0 |
| 434 |
if (!NOutVT.isVector()) { |
0 |
434 |
if (!NOutVT.isVector()) { |
0 |
| 435 |
// For example, i32 = BITCAST v2i16 on alpha. Convert the split |
--- |
435 |
// For example, i32 = BITCAST v2i16 on alpha. Convert the split |
--- |
| 436 |
// pieces of the input into integers and reassemble in the final type. |
--- |
436 |
// pieces of the input into integers and reassemble in the final type. |
--- |
| 437 |
SDValue Lo, Hi; |
0 |
437 |
SDValue Lo, Hi; |
0 |
| 438 |
GetSplitVector(N->getOperand(0), Lo, Hi); |
0 |
438 |
GetSplitVector(N->getOperand(0), Lo, Hi); |
0 |
| 439 |
Lo = BitConvertToInteger(Lo); |
0 |
439 |
Lo = BitConvertToInteger(Lo); |
0 |
| 440 |
Hi = BitConvertToInteger(Hi); |
0 |
440 |
Hi = BitConvertToInteger(Hi); |
0 |
| 441 |
|
--- |
441 |
|
--- |
| 442 |
if (DAG.getDataLayout().isBigEndian()) |
0 |
442 |
if (DAG.getDataLayout().isBigEndian()) |
0 |
| 443 |
std::swap(Lo, Hi); |
0 |
443 |
std::swap(Lo, Hi); |
0 |
| 444 |
|
--- |
444 |
|
--- |
| 445 |
InOp = DAG.getNode(ISD::ANY_EXTEND, dl, |
0 |
445 |
InOp = DAG.getNode(ISD::ANY_EXTEND, dl, |
0 |
| 446 |
EVT::getIntegerVT(*DAG.getContext(), |
0 |
446 |
EVT::getIntegerVT(*DAG.getContext(), |
0 |
| 447 |
NOutVT.getSizeInBits()), |
0 |
447 |
NOutVT.getSizeInBits()), |
0 |
| 448 |
JoinIntegers(Lo, Hi)); |
--- |
448 |
JoinIntegers(Lo, Hi)); |
--- |
| 449 |
return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp); |
0 |
449 |
return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp); |
0 |
| 450 |
} |
--- |
450 |
} |
--- |
| 451 |
break; |
0 |
451 |
break; |
0 |
| 452 |
} |
--- |
452 |
} |
--- |
| 453 |
case TargetLowering::TypeWidenVector: |
0 |
453 |
case TargetLowering::TypeWidenVector: |
0 |
| 454 |
// The input is widened to the same size. Convert to the widened value. |
--- |
454 |
// The input is widened to the same size. Convert to the widened value. |
--- |
| 455 |
// Make sure that the outgoing value is not a vector, because this would |
--- |
455 |
// Make sure that the outgoing value is not a vector, because this would |
--- |
| 456 |
// make us bitcast between two vectors which are legalized in different ways. |
--- |
456 |
// make us bitcast between two vectors which are legalized in different ways. |
--- |
| 457 |
if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector()) { |
0 |
457 |
if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector()) { |
0 |
| 458 |
SDValue Res = |
--- |
458 |
SDValue Res = |
--- |
| 459 |
DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp)); |
0 |
459 |
DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp)); |
0 |
| 460 |
|
--- |
460 |
|
--- |
| 461 |
// For big endian targets we need to shift the casted value or the |
--- |
461 |
// For big endian targets we need to shift the casted value or the |
--- |
| 462 |
// interesting bits will end up at the wrong place. |
--- |
462 |
// interesting bits will end up at the wrong place. |
--- |
| 463 |
if (DAG.getDataLayout().isBigEndian()) { |
0 |
463 |
if (DAG.getDataLayout().isBigEndian()) { |
0 |
| 464 |
unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits(); |
0 |
464 |
unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits(); |
0 |
| 465 |
assert(ShiftAmt < NOutVT.getSizeInBits() && "Too large shift amount!"); |
0 |
465 |
assert(ShiftAmt < NOutVT.getSizeInBits() && "Too large shift amount!"); |
0 |
| 466 |
Res = DAG.getNode(ISD::SRL, dl, NOutVT, Res, |
0 |
466 |
Res = DAG.getNode(ISD::SRL, dl, NOutVT, Res, |
0 |
| 467 |
DAG.getShiftAmountConstant(ShiftAmt, NOutVT, dl)); |
0 |
467 |
DAG.getShiftAmountConstant(ShiftAmt, NOutVT, dl)); |
0 |
| 468 |
} |
--- |
468 |
} |
--- |
| 469 |
return Res; |
0 |
469 |
return Res; |
0 |
| 470 |
} |
--- |
470 |
} |
--- |
| 471 |
// If the output type is also a vector and widening it to the same size |
--- |
471 |
// If the output type is also a vector and widening it to the same size |
--- |
| 472 |
// as the widened input type would be a legal type, we can widen the bitcast |
--- |
472 |
// as the widened input type would be a legal type, we can widen the bitcast |
--- |
| 473 |
// and handle the promotion after. |
--- |
473 |
// and handle the promotion after. |
--- |
| 474 |
if (NOutVT.isVector()) { |
0 |
474 |
if (NOutVT.isVector()) { |
0 |
| 475 |
TypeSize WidenInSize = NInVT.getSizeInBits(); |
0 |
475 |
TypeSize WidenInSize = NInVT.getSizeInBits(); |
0 |
| 476 |
TypeSize OutSize = OutVT.getSizeInBits(); |
0 |
476 |
TypeSize OutSize = OutVT.getSizeInBits(); |
0 |
| 477 |
if (WidenInSize.hasKnownScalarFactor(OutSize)) { |
0 |
477 |
if (WidenInSize.hasKnownScalarFactor(OutSize)) { |
0 |
| 478 |
unsigned Scale = WidenInSize.getKnownScalarFactor(OutSize); |
0 |
478 |
unsigned Scale = WidenInSize.getKnownScalarFactor(OutSize); |
0 |
| 479 |
EVT WideOutVT = |
--- |
479 |
EVT WideOutVT = |
--- |
| 480 |
EVT::getVectorVT(*DAG.getContext(), OutVT.getVectorElementType(), |
0 |
480 |
EVT::getVectorVT(*DAG.getContext(), OutVT.getVectorElementType(), |
0 |
| 481 |
OutVT.getVectorElementCount() * Scale); |
0 |
481 |
OutVT.getVectorElementCount() * Scale); |
0 |
| 482 |
if (isTypeLegal(WideOutVT)) { |
0 |
482 |
if (isTypeLegal(WideOutVT)) { |
0 |
| 483 |
InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp)); |
0 |
483 |
InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp)); |
0 |
| 484 |
InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, InOp, |
0 |
484 |
InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, InOp, |
0 |
| 485 |
DAG.getVectorIdxConstant(0, dl)); |
0 |
485 |
DAG.getVectorIdxConstant(0, dl)); |
0 |
| 486 |
return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, InOp); |
0 |
486 |
return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, InOp); |
0 |
| 487 |
} |
--- |
487 |
} |
--- |
| 488 |
} |
--- |
488 |
} |
--- |
| 489 |
} |
--- |
489 |
} |
--- |
| 490 |
} |
--- |
490 |
} |
--- |
| 491 |
|
--- |
491 |
|
--- |
| 492 |
return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, |
0 |
492 |
return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, |
0 |
| 493 |
CreateStackStoreLoad(InOp, OutVT)); |
0 |
493 |
CreateStackStoreLoad(InOp, OutVT)); |
0 |
| 494 |
} |
0 |
494 |
} |
0 |
| 495 |
|
--- |
495 |
|
--- |
| 496 |
SDValue DAGTypeLegalizer::PromoteIntRes_FREEZE(SDNode *N) { |
0 |
496 |
SDValue DAGTypeLegalizer::PromoteIntRes_FREEZE(SDNode *N) { |
0 |
| 497 |
SDValue V = GetPromotedInteger(N->getOperand(0)); |
0 |
497 |
SDValue V = GetPromotedInteger(N->getOperand(0)); |
0 |
| 498 |
return DAG.getNode(ISD::FREEZE, SDLoc(N), |
0 |
498 |
return DAG.getNode(ISD::FREEZE, SDLoc(N), |
0 |
| 499 |
V.getValueType(), V); |
0 |
499 |
V.getValueType(), V); |
0 |
| 500 |
} |
--- |
500 |
} |
--- |
| 501 |
|
--- |
501 |
|
--- |
| 502 |
SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) { |
0 |
502 |
SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) { |
0 |
| 503 |
SDValue Op = GetPromotedInteger(N->getOperand(0)); |
0 |
503 |
SDValue Op = GetPromotedInteger(N->getOperand(0)); |
0 |
| 504 |
EVT OVT = N->getValueType(0); |
0 |
504 |
EVT OVT = N->getValueType(0); |
0 |
| 505 |
EVT NVT = Op.getValueType(); |
0 |
505 |
EVT NVT = Op.getValueType(); |
0 |
| 506 |
SDLoc dl(N); |
0 |
506 |
SDLoc dl(N); |
0 |
| 507 |
|
--- |
507 |
|
--- |
| 508 |
// If the larger BSWAP isn't supported by the target, try to expand now. |
--- |
508 |
// If the larger BSWAP isn't supported by the target, try to expand now. |
--- |
| 509 |
// If we expand later we'll end up with more operations since we lost the |
--- |
509 |
// If we expand later we'll end up with more operations since we lost the |
--- |
| 510 |
// original type. We only do this for scalars since we have a shuffle |
--- |
510 |
// original type. We only do this for scalars since we have a shuffle |
--- |
| 511 |
// based lowering for vectors in LegalizeVectorOps. |
--- |
511 |
// based lowering for vectors in LegalizeVectorOps. |
--- |
| 512 |
if (!OVT.isVector() && |
0 |
512 |
if (!OVT.isVector() && |
0 |
| 513 |
!TLI.isOperationLegalOrCustomOrPromote(ISD::BSWAP, NVT)) { |
0 |
513 |
!TLI.isOperationLegalOrCustomOrPromote(ISD::BSWAP, NVT)) { |
0 |
| 514 |
if (SDValue Res = TLI.expandBSWAP(N, DAG)) |
0 |
514 |
if (SDValue Res = TLI.expandBSWAP(N, DAG)) |
0 |
| 515 |
return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res); |
0 |
515 |
return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res); |
0 |
| 516 |
} |
--- |
516 |
} |
--- |
| 517 |
|
--- |
517 |
|
--- |
| 518 |
unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(); |
0 |
518 |
unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(); |
0 |
| 519 |
return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op), |
0 |
519 |
return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op), |
0 |
| 520 |
DAG.getShiftAmountConstant(DiffBits, NVT, dl)); |
0 |
520 |
DAG.getShiftAmountConstant(DiffBits, NVT, dl)); |
0 |
| 521 |
} |
0 |
521 |
} |
0 |
| 522 |
|
--- |
522 |
|
--- |
| 523 |
SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) { |
0 |
523 |
SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) { |
0 |
| 524 |
SDValue Op = GetPromotedInteger(N->getOperand(0)); |
0 |
524 |
SDValue Op = GetPromotedInteger(N->getOperand(0)); |
0 |
| 525 |
EVT OVT = N->getValueType(0); |
0 |
525 |
EVT OVT = N->getValueType(0); |
0 |
| 526 |
EVT NVT = Op.getValueType(); |
0 |
526 |
EVT NVT = Op.getValueType(); |
0 |
| 527 |
SDLoc dl(N); |
0 |
527 |
SDLoc dl(N); |
0 |
| 528 |
|
--- |
528 |
|
--- |
| 529 |
// If the larger BITREVERSE isn't supported by the target, try to expand now. |
--- |
529 |
// If the larger BITREVERSE isn't supported by the target, try to expand now. |
--- |
| 530 |
// If we expand later we'll end up with more operations since we lost the |
--- |
530 |
// If we expand later we'll end up with more operations since we lost the |
--- |
| 531 |
// original type. We only do this for scalars since we have a shuffle |
--- |
531 |
// original type. We only do this for scalars since we have a shuffle |
--- |
| 532 |
// based lowering for vectors in LegalizeVectorOps. |
--- |
532 |
// based lowering for vectors in LegalizeVectorOps. |
--- |
| 533 |
if (!OVT.isVector() && OVT.isSimple() && |
0 |
533 |
if (!OVT.isVector() && OVT.isSimple() && |
0 |
| 534 |
!TLI.isOperationLegalOrCustomOrPromote(ISD::BITREVERSE, NVT)) { |
0 |
534 |
!TLI.isOperationLegalOrCustomOrPromote(ISD::BITREVERSE, NVT)) { |
0 |
| 535 |
if (SDValue Res = TLI.expandBITREVERSE(N, DAG)) |
0 |
535 |
if (SDValue Res = TLI.expandBITREVERSE(N, DAG)) |
0 |
| 536 |
return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res); |
0 |
536 |
return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res); |
0 |
| 537 |
} |
--- |
537 |
} |
--- |
| 538 |
|
--- |
538 |
|
--- |
| 539 |
unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(); |
0 |
539 |
unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(); |
0 |
| 540 |
return DAG.getNode(ISD::SRL, dl, NVT, |
0 |
540 |
return DAG.getNode(ISD::SRL, dl, NVT, |
0 |
| 541 |
DAG.getNode(ISD::BITREVERSE, dl, NVT, Op), |
0 |
541 |
DAG.getNode(ISD::BITREVERSE, dl, NVT, Op), |
0 |
| 542 |
DAG.getShiftAmountConstant(DiffBits, NVT, dl)); |
0 |
542 |
DAG.getShiftAmountConstant(DiffBits, NVT, dl)); |
0 |
| 543 |
} |
0 |
543 |
} |
0 |
| 544 |
|
--- |
544 |
|
--- |
| 545 |
SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) { |
0 |
545 |
SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) { |
0 |
| 546 |
// The pair element type may be legal, or may not promote to the same type as |
--- |
546 |
// The pair element type may be legal, or may not promote to the same type as |
--- |
| 547 |
// the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases. |
--- |
547 |
// the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases. |
--- |
| 548 |
return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), |
0 |
548 |
return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), |
0 |
| 549 |
TLI.getTypeToTransformTo(*DAG.getContext(), |
0 |
549 |
TLI.getTypeToTransformTo(*DAG.getContext(), |
0 |
| 550 |
N->getValueType(0)), JoinIntegers(N->getOperand(0), |
0 |
550 |
N->getValueType(0)), JoinIntegers(N->getOperand(0), |
0 |
| 551 |
N->getOperand(1))); |
0 |
551 |
N->getOperand(1))); |
0 |
| 552 |
} |
--- |
552 |
} |
--- |
| 553 |
|
--- |
553 |
|
--- |
| 554 |
SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) { |
0 |
554 |
SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) { |
0 |
| 555 |
EVT VT = N->getValueType(0); |
0 |
555 |
EVT VT = N->getValueType(0); |
0 |
| 556 |
// FIXME there is no actual debug info here |
--- |
556 |
// FIXME there is no actual debug info here |
--- |
| 557 |
SDLoc dl(N); |
0 |
557 |
SDLoc dl(N); |
0 |
| 558 |
// Zero extend things like i1, sign extend everything else. It shouldn't |
--- |
558 |
// Zero extend things like i1, sign extend everything else. It shouldn't |
--- |
| 559 |
// matter in theory which one we pick, but this tends to give better code? |
--- |
559 |
// matter in theory which one we pick, but this tends to give better code? |
--- |
| 560 |
unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; |
0 |
560 |
unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; |
0 |
| 561 |
SDValue Result = DAG.getNode(Opc, dl, |
0 |
561 |
SDValue Result = DAG.getNode(Opc, dl, |
0 |
| 562 |
TLI.getTypeToTransformTo(*DAG.getContext(), VT), |
0 |
562 |
TLI.getTypeToTransformTo(*DAG.getContext(), VT), |
0 |
| 563 |
SDValue(N, 0)); |
--- |
563 |
SDValue(N, 0)); |
--- |
| 564 |
assert(isa(Result) && "Didn't constant fold ext?"); |
0 |
564 |
assert(isa(Result) && "Didn't constant fold ext?"); |
0 |
| 565 |
return Result; |
0 |
565 |
return Result; |
0 |
| 566 |
} |
0 |
566 |
} |
0 |
| 567 |
|
--- |
567 |
|
--- |
| 568 |
SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) { |
0 |
568 |
SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) { |
0 |
| 569 |
EVT OVT = N->getValueType(0); |
0 |
569 |
EVT OVT = N->getValueType(0); |
0 |
| 570 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); |
0 |
570 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); |
0 |
| 571 |
SDLoc dl(N); |
0 |
571 |
SDLoc dl(N); |
0 |
| 572 |
|
--- |
572 |
|
--- |
| 573 |
// If the larger CTLZ isn't supported by the target, try to expand now. |
--- |
573 |
// If the larger CTLZ isn't supported by the target, try to expand now. |
--- |
| 574 |
// If we expand later we'll end up with more operations since we lost the |
--- |
574 |
// If we expand later we'll end up with more operations since we lost the |
--- |
| 575 |
// original type. |
--- |
575 |
// original type. |
--- |
| 576 |
if (!OVT.isVector() && TLI.isTypeLegal(NVT) && |
0 |
576 |
if (!OVT.isVector() && TLI.isTypeLegal(NVT) && |
0 |
| 577 |
!TLI.isOperationLegalOrCustomOrPromote(ISD::CTLZ, NVT) && |
0 |
577 |
!TLI.isOperationLegalOrCustomOrPromote(ISD::CTLZ, NVT) && |
0 |
| 578 |
!TLI.isOperationLegalOrCustomOrPromote(ISD::CTLZ_ZERO_UNDEF, NVT)) { |
0 |
578 |
!TLI.isOperationLegalOrCustomOrPromote(ISD::CTLZ_ZERO_UNDEF, NVT)) { |
0 |
| 579 |
if (SDValue Result = TLI.expandCTLZ(N, DAG)) { |
0 |
579 |
if (SDValue Result = TLI.expandCTLZ(N, DAG)) { |
0 |
| 580 |
Result = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Result); |
0 |
580 |
Result = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Result); |
0 |
| 581 |
return Result; |
0 |
581 |
return Result; |
0 |
| 582 |
} |
--- |
582 |
} |
--- |
| 583 |
} |
--- |
583 |
} |
--- |
| 584 |
|
--- |
584 |
|
--- |
| 585 |
// Zero extend to the promoted type and do the count there. |
--- |
585 |
// Zero extend to the promoted type and do the count there. |
--- |
| 586 |
SDValue Op = ZExtPromotedInteger(N->getOperand(0)); |
0 |
586 |
SDValue Op = ZExtPromotedInteger(N->getOperand(0)); |
0 |
| 587 |
Op = DAG.getNode(N->getOpcode(), dl, NVT, Op); |
0 |
587 |
Op = DAG.getNode(N->getOpcode(), dl, NVT, Op); |
0 |
| 588 |
// Subtract off the extra leading bits in the bigger type. |
--- |
588 |
// Subtract off the extra leading bits in the bigger type. |
--- |
| 589 |
return DAG.getNode( |
0 |
589 |
return DAG.getNode( |
0 |
| 590 |
ISD::SUB, dl, NVT, Op, |
--- |
590 |
ISD::SUB, dl, NVT, Op, |
--- |
| 591 |
DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl, |
0 |
591 |
DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl, |
0 |
| 592 |
NVT)); |
0 |
592 |
NVT)); |
0 |
| 593 |
} |
0 |
593 |
} |
0 |
| 594 |
|
--- |
594 |
|
--- |
| 595 |
SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP_PARITY(SDNode *N) { |
0 |
595 |
SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP_PARITY(SDNode *N) { |
0 |
| 596 |
EVT OVT = N->getValueType(0); |
0 |
596 |
EVT OVT = N->getValueType(0); |
0 |
| 597 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); |
0 |
597 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); |
0 |
| 598 |
|
--- |
598 |
|
--- |
| 599 |
// If the larger CTPOP isn't supported by the target, try to expand now. |
--- |
599 |
// If the larger CTPOP isn't supported by the target, try to expand now. |
--- |
| 600 |
// If we expand later we'll end up with more operations since we lost the |
--- |
600 |
// If we expand later we'll end up with more operations since we lost the |
--- |
| 601 |
// original type. |
--- |
601 |
// original type. |
--- |
| 602 |
// TODO: Expand ISD::PARITY. Need to move ExpandPARITY from LegalizeDAG to |
--- |
602 |
// TODO: Expand ISD::PARITY. Need to move ExpandPARITY from LegalizeDAG to |
--- |
| 603 |
// TargetLowering. |
--- |
603 |
// TargetLowering. |
--- |
| 604 |
if (N->getOpcode() == ISD::CTPOP && !OVT.isVector() && TLI.isTypeLegal(NVT) && |
0 |
604 |
if (N->getOpcode() == ISD::CTPOP && !OVT.isVector() && TLI.isTypeLegal(NVT) && |
0 |
| 605 |
!TLI.isOperationLegalOrCustomOrPromote(ISD::CTPOP, NVT)) { |
0 |
605 |
!TLI.isOperationLegalOrCustomOrPromote(ISD::CTPOP, NVT)) { |
0 |
| 606 |
if (SDValue Result = TLI.expandCTPOP(N, DAG)) { |
0 |
606 |
if (SDValue Result = TLI.expandCTPOP(N, DAG)) { |
0 |
| 607 |
Result = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Result); |
0 |
607 |
Result = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Result); |
0 |
| 608 |
return Result; |
0 |
608 |
return Result; |
0 |
| 609 |
} |
--- |
609 |
} |
--- |
| 610 |
} |
--- |
610 |
} |
--- |
| 611 |
|
--- |
611 |
|
--- |
| 612 |
// Zero extend to the promoted type and do the count or parity there. |
--- |
612 |
// Zero extend to the promoted type and do the count or parity there. |
--- |
| 613 |
SDValue Op = ZExtPromotedInteger(N->getOperand(0)); |
0 |
613 |
SDValue Op = ZExtPromotedInteger(N->getOperand(0)); |
0 |
| 614 |
return DAG.getNode(N->getOpcode(), SDLoc(N), Op.getValueType(), Op); |
0 |
614 |
return DAG.getNode(N->getOpcode(), SDLoc(N), Op.getValueType(), Op); |
0 |
| 615 |
} |
--- |
615 |
} |
--- |
| 616 |
|
--- |
616 |
|
--- |
| 617 |
SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) { |
0 |
617 |
SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) { |
0 |
| 618 |
SDValue Op = GetPromotedInteger(N->getOperand(0)); |
0 |
618 |
SDValue Op = GetPromotedInteger(N->getOperand(0)); |
0 |
| 619 |
EVT OVT = N->getValueType(0); |
0 |
619 |
EVT OVT = N->getValueType(0); |
0 |
| 620 |
EVT NVT = Op.getValueType(); |
0 |
620 |
EVT NVT = Op.getValueType(); |
0 |
| 621 |
SDLoc dl(N); |
0 |
621 |
SDLoc dl(N); |
0 |
| 622 |
|
--- |
622 |
|
--- |
| 623 |
// If the larger CTTZ isn't supported by the target, try to expand now. |
--- |
623 |
// If the larger CTTZ isn't supported by the target, try to expand now. |
--- |
| 624 |
// If we expand later we'll end up with more operations since we lost the |
--- |
624 |
// If we expand later we'll end up with more operations since we lost the |
--- |
| 625 |
// original type. Don't expand if we can use CTPOP or CTLZ expansion on the |
--- |
625 |
// original type. Don't expand if we can use CTPOP or CTLZ expansion on the |
--- |
| 626 |
// larger type. |
--- |
626 |
// larger type. |
--- |
| 627 |
if (!OVT.isVector() && TLI.isTypeLegal(NVT) && |
0 |
627 |
if (!OVT.isVector() && TLI.isTypeLegal(NVT) && |
0 |
| 628 |
!TLI.isOperationLegalOrCustomOrPromote(ISD::CTTZ, NVT) && |
0 |
628 |
!TLI.isOperationLegalOrCustomOrPromote(ISD::CTTZ, NVT) && |
0 |
| 629 |
!TLI.isOperationLegalOrCustomOrPromote(ISD::CTTZ_ZERO_UNDEF, NVT) && |
0 |
629 |
!TLI.isOperationLegalOrCustomOrPromote(ISD::CTTZ_ZERO_UNDEF, NVT) && |
0 |
| 630 |
!TLI.isOperationLegal(ISD::CTPOP, NVT) && |
0 |
630 |
!TLI.isOperationLegal(ISD::CTPOP, NVT) && |
0 |
| 631 |
!TLI.isOperationLegal(ISD::CTLZ, NVT)) { |
0 |
631 |
!TLI.isOperationLegal(ISD::CTLZ, NVT)) { |
0 |
| 632 |
if (SDValue Result = TLI.expandCTTZ(N, DAG)) { |
0 |
632 |
if (SDValue Result = TLI.expandCTTZ(N, DAG)) { |
0 |
| 633 |
Result = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Result); |
0 |
633 |
Result = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Result); |
0 |
| 634 |
return Result; |
0 |
634 |
return Result; |
0 |
| 635 |
} |
--- |
635 |
} |
--- |
| 636 |
} |
--- |
636 |
} |
--- |
| 637 |
|
--- |
637 |
|
--- |
| 638 |
if (N->getOpcode() == ISD::CTTZ) { |
0 |
638 |
if (N->getOpcode() == ISD::CTTZ) { |
0 |
| 639 |
// The count is the same in the promoted type except if the original |
--- |
639 |
// The count is the same in the promoted type except if the original |
--- |
| 640 |
// value was zero. This can be handled by setting the bit just off |
--- |
640 |
// value was zero. This can be handled by setting the bit just off |
--- |
| 641 |
// the top of the original type. |
--- |
641 |
// the top of the original type. |
--- |
| 642 |
auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(), |
0 |
642 |
auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(), |
0 |
| 643 |
OVT.getScalarSizeInBits()); |
0 |
643 |
OVT.getScalarSizeInBits()); |
0 |
| 644 |
Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT)); |
0 |
644 |
Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT)); |
0 |
| 645 |
} |
0 |
645 |
} |
0 |
| 646 |
return DAG.getNode(N->getOpcode(), dl, NVT, Op); |
0 |
646 |
return DAG.getNode(N->getOpcode(), dl, NVT, Op); |
0 |
| 647 |
} |
0 |
647 |
} |
0 |
| 648 |
|
--- |
648 |
|
--- |
| 649 |
SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) { |
0 |
649 |
SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) { |
0 |
| 650 |
SDLoc dl(N); |
0 |
650 |
SDLoc dl(N); |
0 |
| 651 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
651 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
| 652 |
|
--- |
652 |
|
--- |
| 653 |
SDValue Op0 = N->getOperand(0); |
0 |
653 |
SDValue Op0 = N->getOperand(0); |
0 |
| 654 |
SDValue Op1 = N->getOperand(1); |
0 |
654 |
SDValue Op1 = N->getOperand(1); |
0 |
| 655 |
|
--- |
655 |
|
--- |
| 656 |
// If the input also needs to be promoted, do that first so we can get a |
--- |
656 |
// If the input also needs to be promoted, do that first so we can get a |
--- |
| 657 |
// get a good idea for the output type. |
--- |
657 |
// get a good idea for the output type. |
--- |
| 658 |
if (TLI.getTypeAction(*DAG.getContext(), Op0.getValueType()) |
0 |
658 |
if (TLI.getTypeAction(*DAG.getContext(), Op0.getValueType()) |
0 |
| 659 |
== TargetLowering::TypePromoteInteger) { |
0 |
659 |
== TargetLowering::TypePromoteInteger) { |
0 |
| 660 |
SDValue In = GetPromotedInteger(Op0); |
0 |
660 |
SDValue In = GetPromotedInteger(Op0); |
0 |
| 661 |
|
--- |
661 |
|
--- |
| 662 |
// If the new type is larger than NVT, use it. We probably won't need to |
--- |
662 |
// If the new type is larger than NVT, use it. We probably won't need to |
--- |
| 663 |
// promote it again. |
--- |
663 |
// promote it again. |
--- |
| 664 |
EVT SVT = In.getValueType().getScalarType(); |
0 |
664 |
EVT SVT = In.getValueType().getScalarType(); |
0 |
| 665 |
if (SVT.bitsGE(NVT)) { |
0 |
665 |
if (SVT.bitsGE(NVT)) { |
0 |
| 666 |
SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, In, Op1); |
0 |
666 |
SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, In, Op1); |
0 |
| 667 |
return DAG.getAnyExtOrTrunc(Ext, dl, NVT); |
0 |
667 |
return DAG.getAnyExtOrTrunc(Ext, dl, NVT); |
0 |
| 668 |
} |
--- |
668 |
} |
--- |
| 669 |
} |
--- |
669 |
} |
--- |
| 670 |
|
--- |
670 |
|
--- |
| 671 |
return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, Op0, Op1); |
0 |
671 |
return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, Op0, Op1); |
0 |
| 672 |
} |
0 |
672 |
} |
0 |
| 673 |
|
--- |
673 |
|
--- |
| 674 |
SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) { |
0 |
674 |
SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) { |
0 |
| 675 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
675 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
| 676 |
unsigned NewOpc = N->getOpcode(); |
0 |
676 |
unsigned NewOpc = N->getOpcode(); |
0 |
| 677 |
SDLoc dl(N); |
0 |
677 |
SDLoc dl(N); |
0 |
| 678 |
|
--- |
678 |
|
--- |
| 679 |
// If we're promoting a UINT to a larger size and the larger FP_TO_UINT is |
--- |
679 |
// If we're promoting a UINT to a larger size and the larger FP_TO_UINT is |
--- |
| 680 |
// not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT |
--- |
680 |
// not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT |
--- |
| 681 |
// and SINT conversions are Custom, there is no way to tell which is |
--- |
681 |
// and SINT conversions are Custom, there is no way to tell which is |
--- |
| 682 |
// preferable. We choose SINT because that's the right thing on PPC.) |
--- |
682 |
// preferable. We choose SINT because that's the right thing on PPC.) |
--- |
| 683 |
if (N->getOpcode() == ISD::FP_TO_UINT && |
0 |
683 |
if (N->getOpcode() == ISD::FP_TO_UINT && |
0 |
| 684 |
!TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) && |
0 |
684 |
!TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) && |
0 |
| 685 |
TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT)) |
0 |
685 |
TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT)) |
0 |
| 686 |
NewOpc = ISD::FP_TO_SINT; |
0 |
686 |
NewOpc = ISD::FP_TO_SINT; |
0 |
| 687 |
|
--- |
687 |
|
--- |
| 688 |
if (N->getOpcode() == ISD::STRICT_FP_TO_UINT && |
0 |
688 |
if (N->getOpcode() == ISD::STRICT_FP_TO_UINT && |
0 |
| 689 |
!TLI.isOperationLegal(ISD::STRICT_FP_TO_UINT, NVT) && |
0 |
689 |
!TLI.isOperationLegal(ISD::STRICT_FP_TO_UINT, NVT) && |
0 |
| 690 |
TLI.isOperationLegalOrCustom(ISD::STRICT_FP_TO_SINT, NVT)) |
0 |
690 |
TLI.isOperationLegalOrCustom(ISD::STRICT_FP_TO_SINT, NVT)) |
0 |
| 691 |
NewOpc = ISD::STRICT_FP_TO_SINT; |
0 |
691 |
NewOpc = ISD::STRICT_FP_TO_SINT; |
0 |
| 692 |
|
--- |
692 |
|
--- |
| 693 |
if (N->getOpcode() == ISD::VP_FP_TO_UINT && |
0 |
693 |
if (N->getOpcode() == ISD::VP_FP_TO_UINT && |
0 |
| 694 |
!TLI.isOperationLegal(ISD::VP_FP_TO_UINT, NVT) && |
0 |
694 |
!TLI.isOperationLegal(ISD::VP_FP_TO_UINT, NVT) && |
0 |
| 695 |
TLI.isOperationLegalOrCustom(ISD::VP_FP_TO_SINT, NVT)) |
0 |
695 |
TLI.isOperationLegalOrCustom(ISD::VP_FP_TO_SINT, NVT)) |
0 |
| 696 |
NewOpc = ISD::VP_FP_TO_SINT; |
0 |
696 |
NewOpc = ISD::VP_FP_TO_SINT; |
0 |
| 697 |
|
--- |
697 |
|
--- |
| 698 |
SDValue Res; |
0 |
698 |
SDValue Res; |
0 |
| 699 |
if (N->isStrictFPOpcode()) { |
0 |
699 |
if (N->isStrictFPOpcode()) { |
0 |
| 700 |
Res = DAG.getNode(NewOpc, dl, {NVT, MVT::Other}, |
0 |
700 |
Res = DAG.getNode(NewOpc, dl, {NVT, MVT::Other}, |
0 |
| 701 |
{N->getOperand(0), N->getOperand(1)}); |
0 |
701 |
{N->getOperand(0), N->getOperand(1)}); |
0 |
| 702 |
// Legalize the chain result - switch anything that used the old chain to |
--- |
702 |
// Legalize the chain result - switch anything that used the old chain to |
--- |
| 703 |
// use the new one. |
--- |
703 |
// use the new one. |
--- |
| 704 |
ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); |
0 |
704 |
ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); |
0 |
| 705 |
} else if (NewOpc == ISD::VP_FP_TO_SINT || NewOpc == ISD::VP_FP_TO_UINT) { |
0 |
705 |
} else if (NewOpc == ISD::VP_FP_TO_SINT || NewOpc == ISD::VP_FP_TO_UINT) { |
0 |
| 706 |
Res = DAG.getNode(NewOpc, dl, NVT, {N->getOperand(0), N->getOperand(1), |
0 |
706 |
Res = DAG.getNode(NewOpc, dl, NVT, {N->getOperand(0), N->getOperand(1), |
0 |
| 707 |
N->getOperand(2)}); |
0 |
707 |
N->getOperand(2)}); |
0 |
| 708 |
} else { |
--- |
708 |
} else { |
--- |
| 709 |
Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0)); |
0 |
709 |
Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0)); |
0 |
| 710 |
} |
--- |
710 |
} |
--- |
| 711 |
|
--- |
711 |
|
--- |
| 712 |
// Assert that the converted value fits in the original type. If it doesn't |
--- |
712 |
// Assert that the converted value fits in the original type. If it doesn't |
--- |
| 713 |
// (eg: because the value being converted is too big), then the result of the |
--- |
713 |
// (eg: because the value being converted is too big), then the result of the |
--- |
| 714 |
// original operation was undefined anyway, so the assert is still correct. |
--- |
714 |
// original operation was undefined anyway, so the assert is still correct. |
--- |
| 715 |
// |
--- |
715 |
// |
--- |
| 716 |
// NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example: |
--- |
716 |
// NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example: |
--- |
| 717 |
// before legalization: fp-to-uint16, 65534. -> 0xfffe |
--- |
717 |
// before legalization: fp-to-uint16, 65534. -> 0xfffe |
--- |
| 718 |
// after legalization: fp-to-sint32, 65534. -> 0x0000fffe |
--- |
718 |
// after legalization: fp-to-sint32, 65534. -> 0x0000fffe |
--- |
| 719 |
return DAG.getNode((N->getOpcode() == ISD::FP_TO_UINT || |
0 |
719 |
return DAG.getNode((N->getOpcode() == ISD::FP_TO_UINT || |
0 |
| 720 |
N->getOpcode() == ISD::STRICT_FP_TO_UINT || |
0 |
720 |
N->getOpcode() == ISD::STRICT_FP_TO_UINT || |
0 |
| 721 |
N->getOpcode() == ISD::VP_FP_TO_UINT) |
0 |
721 |
N->getOpcode() == ISD::VP_FP_TO_UINT) |
0 |
| 722 |
? ISD::AssertZext |
--- |
722 |
? ISD::AssertZext |
--- |
| 723 |
: ISD::AssertSext, |
--- |
723 |
: ISD::AssertSext, |
--- |
| 724 |
dl, NVT, Res, |
--- |
724 |
dl, NVT, Res, |
--- |
| 725 |
DAG.getValueType(N->getValueType(0).getScalarType())); |
0 |
725 |
DAG.getValueType(N->getValueType(0).getScalarType())); |
0 |
| 726 |
} |
0 |
726 |
} |
0 |
| 727 |
|
--- |
727 |
|
--- |
| 728 |
SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(SDNode *N) { |
0 |
728 |
SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(SDNode *N) { |
0 |
| 729 |
// Promote the result type, while keeping the original width in Op1. |
--- |
729 |
// Promote the result type, while keeping the original width in Op1. |
--- |
| 730 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
730 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
| 731 |
SDLoc dl(N); |
0 |
731 |
SDLoc dl(N); |
0 |
| 732 |
return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0), |
0 |
732 |
return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0), |
0 |
| 733 |
N->getOperand(1)); |
0 |
733 |
N->getOperand(1)); |
0 |
| 734 |
} |
0 |
734 |
} |
0 |
| 735 |
|
--- |
735 |
|
--- |
| 736 |
SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16_BF16(SDNode *N) { |
0 |
736 |
SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16_BF16(SDNode *N) { |
0 |
| 737 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
737 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
| 738 |
SDLoc dl(N); |
0 |
738 |
SDLoc dl(N); |
0 |
| 739 |
|
--- |
739 |
|
--- |
| 740 |
return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); |
0 |
740 |
return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); |
0 |
| 741 |
} |
0 |
741 |
} |
0 |
| 742 |
|
--- |
742 |
|
--- |
| 743 |
SDValue DAGTypeLegalizer::PromoteIntRes_GET_ROUNDING(SDNode *N) { |
0 |
743 |
SDValue DAGTypeLegalizer::PromoteIntRes_GET_ROUNDING(SDNode *N) { |
0 |
| 744 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
744 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
| 745 |
SDLoc dl(N); |
0 |
745 |
SDLoc dl(N); |
0 |
| 746 |
|
--- |
746 |
|
--- |
| 747 |
SDValue Res = |
--- |
747 |
SDValue Res = |
--- |
| 748 |
DAG.getNode(N->getOpcode(), dl, {NVT, MVT::Other}, N->getOperand(0)); |
0 |
748 |
DAG.getNode(N->getOpcode(), dl, {NVT, MVT::Other}, N->getOperand(0)); |
0 |
| 749 |
|
--- |
749 |
|
--- |
| 750 |
// Legalize the chain result - switch anything that used the old chain to |
--- |
750 |
// Legalize the chain result - switch anything that used the old chain to |
--- |
| 751 |
// use the new one. |
--- |
751 |
// use the new one. |
--- |
| 752 |
ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); |
0 |
752 |
ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); |
0 |
| 753 |
return Res; |
0 |
753 |
return Res; |
0 |
| 754 |
} |
0 |
754 |
} |
0 |
| 755 |
|
--- |
755 |
|
--- |
| 756 |
SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) { |
0 |
756 |
SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) { |
0 |
| 757 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
757 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
| 758 |
SDLoc dl(N); |
0 |
758 |
SDLoc dl(N); |
0 |
| 759 |
|
--- |
759 |
|
--- |
| 760 |
if (getTypeAction(N->getOperand(0).getValueType()) |
0 |
760 |
if (getTypeAction(N->getOperand(0).getValueType()) |
0 |
| 761 |
== TargetLowering::TypePromoteInteger) { |
0 |
761 |
== TargetLowering::TypePromoteInteger) { |
0 |
| 762 |
SDValue Res = GetPromotedInteger(N->getOperand(0)); |
0 |
762 |
SDValue Res = GetPromotedInteger(N->getOperand(0)); |
0 |
| 763 |
assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!"); |
0 |
763 |
assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!"); |
0 |
| 764 |
|
--- |
764 |
|
--- |
| 765 |
// If the result and operand types are the same after promotion, simplify |
--- |
765 |
// If the result and operand types are the same after promotion, simplify |
--- |
| 766 |
// to an in-register extension. Unless this is a VP_*_EXTEND. |
--- |
766 |
// to an in-register extension. Unless this is a VP_*_EXTEND. |
--- |
| 767 |
if (NVT == Res.getValueType() && N->getNumOperands() == 1) { |
0 |
767 |
if (NVT == Res.getValueType() && N->getNumOperands() == 1) { |
0 |
| 768 |
// The high bits are not guaranteed to be anything. Insert an extend. |
--- |
768 |
// The high bits are not guaranteed to be anything. Insert an extend. |
--- |
| 769 |
if (N->getOpcode() == ISD::SIGN_EXTEND) |
0 |
769 |
if (N->getOpcode() == ISD::SIGN_EXTEND) |
0 |
| 770 |
return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, |
0 |
770 |
return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, |
0 |
| 771 |
DAG.getValueType(N->getOperand(0).getValueType())); |
0 |
771 |
DAG.getValueType(N->getOperand(0).getValueType())); |
0 |
| 772 |
if (N->getOpcode() == ISD::ZERO_EXTEND) |
0 |
772 |
if (N->getOpcode() == ISD::ZERO_EXTEND) |
0 |
| 773 |
return DAG.getZeroExtendInReg(Res, dl, N->getOperand(0).getValueType()); |
0 |
773 |
return DAG.getZeroExtendInReg(Res, dl, N->getOperand(0).getValueType()); |
0 |
| 774 |
assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!"); |
0 |
774 |
assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!"); |
0 |
| 775 |
return Res; |
0 |
775 |
return Res; |
0 |
| 776 |
} |
--- |
776 |
} |
--- |
| 777 |
} |
--- |
777 |
} |
--- |
| 778 |
|
--- |
778 |
|
--- |
| 779 |
// Otherwise, just extend the original operand all the way to the larger type. |
--- |
779 |
// Otherwise, just extend the original operand all the way to the larger type. |
--- |
| 780 |
if (N->getNumOperands() != 1) { |
0 |
780 |
if (N->getNumOperands() != 1) { |
0 |
| 781 |
assert(N->getNumOperands() == 3 && "Unexpected number of operands!"); |
0 |
781 |
assert(N->getNumOperands() == 3 && "Unexpected number of operands!"); |
0 |
| 782 |
assert(N->isVPOpcode() && "Expected VP opcode"); |
0 |
782 |
assert(N->isVPOpcode() && "Expected VP opcode"); |
0 |
| 783 |
return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0), |
0 |
783 |
return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0), |
0 |
| 784 |
N->getOperand(1), N->getOperand(2)); |
0 |
784 |
N->getOperand(1), N->getOperand(2)); |
0 |
| 785 |
} |
--- |
785 |
} |
--- |
| 786 |
return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); |
0 |
786 |
return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); |
0 |
| 787 |
} |
0 |
787 |
} |
0 |
| 788 |
|
--- |
788 |
|
--- |
| 789 |
SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) { |
0 |
789 |
SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) { |
0 |
| 790 |
assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); |
0 |
790 |
assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); |
0 |
| 791 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
791 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
| 792 |
ISD::LoadExtType ExtType = |
--- |
792 |
ISD::LoadExtType ExtType = |
--- |
| 793 |
ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType(); |
0 |
793 |
ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType(); |
0 |
| 794 |
SDLoc dl(N); |
0 |
794 |
SDLoc dl(N); |
0 |
| 795 |
SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(), |
0 |
795 |
SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(), |
0 |
| 796 |
N->getMemoryVT(), N->getMemOperand()); |
--- |
796 |
N->getMemoryVT(), N->getMemOperand()); |
--- |
| 797 |
|
--- |
797 |
|
--- |
| 798 |
// Legalize the chain result - switch anything that used the old chain to |
--- |
798 |
// Legalize the chain result - switch anything that used the old chain to |
--- |
| 799 |
// use the new one. |
--- |
799 |
// use the new one. |
--- |
| 800 |
ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); |
0 |
800 |
ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); |
0 |
| 801 |
return Res; |
0 |
801 |
return Res; |
0 |
| 802 |
} |
0 |
802 |
} |
0 |
| 803 |
|
--- |
803 |
|
--- |
| 804 |
SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) { |
0 |
804 |
SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) { |
0 |
| 805 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
805 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
| 806 |
SDValue ExtPassThru = GetPromotedInteger(N->getPassThru()); |
0 |
806 |
SDValue ExtPassThru = GetPromotedInteger(N->getPassThru()); |
0 |
| 807 |
|
--- |
807 |
|
--- |
| 808 |
ISD::LoadExtType ExtType = N->getExtensionType(); |
0 |
808 |
ISD::LoadExtType ExtType = N->getExtensionType(); |
0 |
| 809 |
if (ExtType == ISD::NON_EXTLOAD) |
0 |
809 |
if (ExtType == ISD::NON_EXTLOAD) |
0 |
| 810 |
ExtType = ISD::EXTLOAD; |
0 |
810 |
ExtType = ISD::EXTLOAD; |
0 |
| 811 |
|
--- |
811 |
|
--- |
| 812 |
SDLoc dl(N); |
0 |
812 |
SDLoc dl(N); |
0 |
| 813 |
SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(), |
0 |
813 |
SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(), |
0 |
| 814 |
N->getOffset(), N->getMask(), ExtPassThru, |
0 |
814 |
N->getOffset(), N->getMask(), ExtPassThru, |
0 |
| 815 |
N->getMemoryVT(), N->getMemOperand(), |
--- |
815 |
N->getMemoryVT(), N->getMemOperand(), |
--- |
| 816 |
N->getAddressingMode(), ExtType, |
--- |
816 |
N->getAddressingMode(), ExtType, |
--- |
| 817 |
N->isExpandingLoad()); |
0 |
817 |
N->isExpandingLoad()); |
0 |
| 818 |
// Legalize the chain result - switch anything that used the old chain to |
--- |
818 |
// Legalize the chain result - switch anything that used the old chain to |
--- |
| 819 |
// use the new one. |
--- |
819 |
// use the new one. |
--- |
| 820 |
ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); |
0 |
820 |
ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); |
0 |
| 821 |
return Res; |
0 |
821 |
return Res; |
0 |
| 822 |
} |
0 |
822 |
} |
0 |
| 823 |
|
--- |
823 |
|
--- |
| 824 |
SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) { |
0 |
824 |
SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) { |
0 |
| 825 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
825 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
| 826 |
SDValue ExtPassThru = GetPromotedInteger(N->getPassThru()); |
0 |
826 |
SDValue ExtPassThru = GetPromotedInteger(N->getPassThru()); |
0 |
| 827 |
assert(NVT == ExtPassThru.getValueType() && |
0 |
827 |
assert(NVT == ExtPassThru.getValueType() && |
0 |
| 828 |
"Gather result type and the passThru argument type should be the same"); |
--- |
828 |
"Gather result type and the passThru argument type should be the same"); |
--- |
| 829 |
|
--- |
829 |
|
--- |
| 830 |
ISD::LoadExtType ExtType = N->getExtensionType(); |
0 |
830 |
ISD::LoadExtType ExtType = N->getExtensionType(); |
0 |
| 831 |
if (ExtType == ISD::NON_EXTLOAD) |
0 |
831 |
if (ExtType == ISD::NON_EXTLOAD) |
0 |
| 832 |
ExtType = ISD::EXTLOAD; |
0 |
832 |
ExtType = ISD::EXTLOAD; |
0 |
| 833 |
|
--- |
833 |
|
--- |
| 834 |
SDLoc dl(N); |
0 |
834 |
SDLoc dl(N); |
0 |
| 835 |
SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(), |
0 |
835 |
SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(), |
0 |
| 836 |
N->getIndex(), N->getScale() }; |
0 |
836 |
N->getIndex(), N->getScale() }; |
0 |
| 837 |
SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other), |
0 |
837 |
SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other), |
0 |
| 838 |
N->getMemoryVT(), dl, Ops, |
--- |
838 |
N->getMemoryVT(), dl, Ops, |
--- |
| 839 |
N->getMemOperand(), N->getIndexType(), |
--- |
839 |
N->getMemOperand(), N->getIndexType(), |
--- |
| 840 |
ExtType); |
--- |
840 |
ExtType); |
--- |
| 841 |
// Legalize the chain result - switch anything that used the old chain to |
--- |
841 |
// Legalize the chain result - switch anything that used the old chain to |
--- |
| 842 |
// use the new one. |
--- |
842 |
// use the new one. |
--- |
| 843 |
ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); |
0 |
843 |
ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); |
0 |
| 844 |
return Res; |
0 |
844 |
return Res; |
0 |
| 845 |
} |
0 |
845 |
} |
0 |
| 846 |
|
--- |
846 |
|
--- |
| 847 |
/// Promote the overflow flag of an overflowing arithmetic node. |
--- |
847 |
/// Promote the overflow flag of an overflowing arithmetic node. |
--- |
| 848 |
SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) { |
0 |
848 |
SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) { |
0 |
| 849 |
// Change the return type of the boolean result while obeying |
--- |
849 |
// Change the return type of the boolean result while obeying |
--- |
| 850 |
// getSetCCResultType. |
--- |
850 |
// getSetCCResultType. |
--- |
| 851 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); |
0 |
851 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); |
0 |
| 852 |
EVT VT = N->getValueType(0); |
0 |
852 |
EVT VT = N->getValueType(0); |
0 |
| 853 |
EVT SVT = getSetCCResultType(VT); |
0 |
853 |
EVT SVT = getSetCCResultType(VT); |
0 |
| 854 |
SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) }; |
0 |
854 |
SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) }; |
0 |
| 855 |
unsigned NumOps = N->getNumOperands(); |
0 |
855 |
unsigned NumOps = N->getNumOperands(); |
0 |
| 856 |
assert(NumOps <= 3 && "Too many operands"); |
0 |
856 |
assert(NumOps <= 3 && "Too many operands"); |
0 |
| 857 |
if (NumOps == 3) |
0 |
857 |
if (NumOps == 3) |
0 |
| 858 |
Ops[2] = N->getOperand(2); |
0 |
858 |
Ops[2] = N->getOperand(2); |
0 |
| 859 |
|
--- |
859 |
|
--- |
| 860 |
SDLoc dl(N); |
0 |
860 |
SDLoc dl(N); |
0 |
| 861 |
SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, SVT), |
0 |
861 |
SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, SVT), |
0 |
| 862 |
ArrayRef(Ops, NumOps)); |
--- |
862 |
ArrayRef(Ops, NumOps)); |
--- |
| 863 |
|
--- |
863 |
|
--- |
| 864 |
// Modified the sum result - switch anything that used the old sum to use |
--- |
864 |
// Modified the sum result - switch anything that used the old sum to use |
--- |
| 865 |
// the new one. |
--- |
865 |
// the new one. |
--- |
| 866 |
ReplaceValueWith(SDValue(N, 0), Res); |
0 |
866 |
ReplaceValueWith(SDValue(N, 0), Res); |
0 |
| 867 |
|
--- |
867 |
|
--- |
| 868 |
// Convert to the expected type. |
--- |
868 |
// Convert to the expected type. |
--- |
| 869 |
return DAG.getBoolExtOrTrunc(Res.getValue(1), dl, NVT, VT); |
0 |
869 |
return DAG.getBoolExtOrTrunc(Res.getValue(1), dl, NVT, VT); |
0 |
| 870 |
} |
0 |
870 |
} |
0 |
| 871 |
|
--- |
871 |
|
--- |
| 872 |
SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSHLSAT(SDNode *N) { |
0 |
872 |
SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSHLSAT(SDNode *N) { |
0 |
| 873 |
// If the promoted type is legal, we can convert this to: |
--- |
873 |
// If the promoted type is legal, we can convert this to: |
--- |
| 874 |
// 1. ANY_EXTEND iN to iM |
--- |
874 |
// 1. ANY_EXTEND iN to iM |
--- |
| 875 |
// 2. SHL by M-N |
--- |
875 |
// 2. SHL by M-N |
--- |
| 876 |
// 3. [US][ADD|SUB|SHL]SAT |
--- |
876 |
// 3. [US][ADD|SUB|SHL]SAT |
--- |
| 877 |
// 4. L/ASHR by M-N |
--- |
877 |
// 4. L/ASHR by M-N |
--- |
| 878 |
// Else it is more efficient to convert this to a min and a max |
--- |
878 |
// Else it is more efficient to convert this to a min and a max |
--- |
| 879 |
// operation in the higher precision arithmetic. |
--- |
879 |
// operation in the higher precision arithmetic. |
--- |
| 880 |
SDLoc dl(N); |
0 |
880 |
SDLoc dl(N); |
0 |
| 881 |
SDValue Op1 = N->getOperand(0); |
0 |
881 |
SDValue Op1 = N->getOperand(0); |
0 |
| 882 |
SDValue Op2 = N->getOperand(1); |
0 |
882 |
SDValue Op2 = N->getOperand(1); |
0 |
| 883 |
unsigned OldBits = Op1.getScalarValueSizeInBits(); |
0 |
883 |
unsigned OldBits = Op1.getScalarValueSizeInBits(); |
0 |
| 884 |
|
--- |
884 |
|
--- |
| 885 |
unsigned Opcode = N->getOpcode(); |
0 |
885 |
unsigned Opcode = N->getOpcode(); |
0 |
| 886 |
bool IsShift = Opcode == ISD::USHLSAT || Opcode == ISD::SSHLSAT; |
0 |
886 |
bool IsShift = Opcode == ISD::USHLSAT || Opcode == ISD::SSHLSAT; |
0 |
| 887 |
|
--- |
887 |
|
--- |
| 888 |
SDValue Op1Promoted, Op2Promoted; |
0 |
888 |
SDValue Op1Promoted, Op2Promoted; |
0 |
| 889 |
if (IsShift) { |
0 |
889 |
if (IsShift) { |
0 |
| 890 |
Op1Promoted = GetPromotedInteger(Op1); |
0 |
890 |
Op1Promoted = GetPromotedInteger(Op1); |
0 |
| 891 |
Op2Promoted = ZExtPromotedInteger(Op2); |
0 |
891 |
Op2Promoted = ZExtPromotedInteger(Op2); |
0 |
| 892 |
} else if (Opcode == ISD::UADDSAT || Opcode == ISD::USUBSAT) { |
0 |
892 |
} else if (Opcode == ISD::UADDSAT || Opcode == ISD::USUBSAT) { |
0 |
| 893 |
Op1Promoted = ZExtPromotedInteger(Op1); |
0 |
893 |
Op1Promoted = ZExtPromotedInteger(Op1); |
0 |
| 894 |
Op2Promoted = ZExtPromotedInteger(Op2); |
0 |
894 |
Op2Promoted = ZExtPromotedInteger(Op2); |
0 |
| 895 |
} else { |
--- |
895 |
} else { |
--- |
| 896 |
Op1Promoted = SExtPromotedInteger(Op1); |
0 |
896 |
Op1Promoted = SExtPromotedInteger(Op1); |
0 |
| 897 |
Op2Promoted = SExtPromotedInteger(Op2); |
0 |
897 |
Op2Promoted = SExtPromotedInteger(Op2); |
0 |
| 898 |
} |
--- |
898 |
} |
--- |
| 899 |
EVT PromotedType = Op1Promoted.getValueType(); |
0 |
899 |
EVT PromotedType = Op1Promoted.getValueType(); |
0 |
| 900 |
unsigned NewBits = PromotedType.getScalarSizeInBits(); |
0 |
900 |
unsigned NewBits = PromotedType.getScalarSizeInBits(); |
0 |
| 901 |
|
--- |
901 |
|
--- |
| 902 |
if (Opcode == ISD::UADDSAT) { |
0 |
902 |
if (Opcode == ISD::UADDSAT) { |
0 |
| 903 |
APInt MaxVal = APInt::getAllOnes(OldBits).zext(NewBits); |
0 |
903 |
APInt MaxVal = APInt::getAllOnes(OldBits).zext(NewBits); |
0 |
| 904 |
SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType); |
0 |
904 |
SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType); |
0 |
| 905 |
SDValue Add = |
--- |
905 |
SDValue Add = |
--- |
| 906 |
DAG.getNode(ISD::ADD, dl, PromotedType, Op1Promoted, Op2Promoted); |
0 |
906 |
DAG.getNode(ISD::ADD, dl, PromotedType, Op1Promoted, Op2Promoted); |
0 |
| 907 |
return DAG.getNode(ISD::UMIN, dl, PromotedType, Add, SatMax); |
0 |
907 |
return DAG.getNode(ISD::UMIN, dl, PromotedType, Add, SatMax); |
0 |
| 908 |
} |
0 |
908 |
} |
0 |
| 909 |
|
--- |
909 |
|
--- |
| 910 |
// USUBSAT can always be promoted as long as we have zero-extended the args. |
--- |
910 |
// USUBSAT can always be promoted as long as we have zero-extended the args. |
--- |
| 911 |
if (Opcode == ISD::USUBSAT) |
0 |
911 |
if (Opcode == ISD::USUBSAT) |
0 |
| 912 |
return DAG.getNode(ISD::USUBSAT, dl, PromotedType, Op1Promoted, |
0 |
912 |
return DAG.getNode(ISD::USUBSAT, dl, PromotedType, Op1Promoted, |
0 |
| 913 |
Op2Promoted); |
0 |
913 |
Op2Promoted); |
0 |
| 914 |
|
--- |
914 |
|
--- |
| 915 |
// Shift cannot use a min/max expansion, we can't detect overflow if all of |
--- |
915 |
// Shift cannot use a min/max expansion, we can't detect overflow if all of |
--- |
| 916 |
// the bits have been shifted out. |
--- |
916 |
// the bits have been shifted out. |
--- |
| 917 |
if (IsShift || TLI.isOperationLegal(Opcode, PromotedType)) { |
0 |
917 |
if (IsShift || TLI.isOperationLegal(Opcode, PromotedType)) { |
0 |
| 918 |
unsigned ShiftOp; |
--- |
918 |
unsigned ShiftOp; |
--- |
| 919 |
switch (Opcode) { |
0 |
919 |
switch (Opcode) { |
0 |
| 920 |
case ISD::SADDSAT: |
0 |
920 |
case ISD::SADDSAT: |
0 |
| 921 |
case ISD::SSUBSAT: |
--- |
921 |
case ISD::SSUBSAT: |
--- |
| 922 |
case ISD::SSHLSAT: |
--- |
922 |
case ISD::SSHLSAT: |
--- |
| 923 |
ShiftOp = ISD::SRA; |
0 |
923 |
ShiftOp = ISD::SRA; |
0 |
| 924 |
break; |
0 |
924 |
break; |
0 |
| 925 |
case ISD::USHLSAT: |
0 |
925 |
case ISD::USHLSAT: |
0 |
| 926 |
ShiftOp = ISD::SRL; |
0 |
926 |
ShiftOp = ISD::SRL; |
0 |
| 927 |
break; |
0 |
927 |
break; |
0 |
| 928 |
default: |
0 |
928 |
default: |
0 |
| 929 |
llvm_unreachable("Expected opcode to be signed or unsigned saturation " |
0 |
929 |
llvm_unreachable("Expected opcode to be signed or unsigned saturation " |
0 |
| 930 |
"addition, subtraction or left shift"); |
--- |
930 |
"addition, subtraction or left shift"); |
--- |
| 931 |
} |
--- |
931 |
} |
--- |
| 932 |
|
--- |
932 |
|
--- |
| 933 |
unsigned SHLAmount = NewBits - OldBits; |
0 |
933 |
unsigned SHLAmount = NewBits - OldBits; |
0 |
| 934 |
SDValue ShiftAmount = |
--- |
934 |
SDValue ShiftAmount = |
--- |
| 935 |
DAG.getShiftAmountConstant(SHLAmount, PromotedType, dl); |
0 |
935 |
DAG.getShiftAmountConstant(SHLAmount, PromotedType, dl); |
0 |
| 936 |
Op1Promoted = |
0 |
936 |
Op1Promoted = |
0 |
| 937 |
DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount); |
0 |
937 |
DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount); |
0 |
| 938 |
if (!IsShift) |
0 |
938 |
if (!IsShift) |
0 |
| 939 |
Op2Promoted = |
0 |
939 |
Op2Promoted = |
0 |
| 940 |
DAG.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount); |
0 |
940 |
DAG.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount); |
0 |
| 941 |
|
--- |
941 |
|
--- |
| 942 |
SDValue Result = |
--- |
942 |
SDValue Result = |
--- |
| 943 |
DAG.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted); |
0 |
943 |
DAG.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted); |
0 |
| 944 |
return DAG.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount); |
0 |
944 |
return DAG.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount); |
0 |
| 945 |
} |
--- |
945 |
} |
--- |
| 946 |
|
--- |
946 |
|
--- |
| 947 |
unsigned AddOp = Opcode == ISD::SADDSAT ? ISD::ADD : ISD::SUB; |
0 |
947 |
unsigned AddOp = Opcode == ISD::SADDSAT ? ISD::ADD : ISD::SUB; |
0 |
| 948 |
APInt MinVal = APInt::getSignedMinValue(OldBits).sext(NewBits); |
0 |
948 |
APInt MinVal = APInt::getSignedMinValue(OldBits).sext(NewBits); |
0 |
| 949 |
APInt MaxVal = APInt::getSignedMaxValue(OldBits).sext(NewBits); |
0 |
949 |
APInt MaxVal = APInt::getSignedMaxValue(OldBits).sext(NewBits); |
0 |
| 950 |
SDValue SatMin = DAG.getConstant(MinVal, dl, PromotedType); |
0 |
950 |
SDValue SatMin = DAG.getConstant(MinVal, dl, PromotedType); |
0 |
| 951 |
SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType); |
0 |
951 |
SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType); |
0 |
| 952 |
SDValue Result = |
--- |
952 |
SDValue Result = |
--- |
| 953 |
DAG.getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted); |
0 |
953 |
DAG.getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted); |
0 |
| 954 |
Result = DAG.getNode(ISD::SMIN, dl, PromotedType, Result, SatMax); |
0 |
954 |
Result = DAG.getNode(ISD::SMIN, dl, PromotedType, Result, SatMax); |
0 |
| 955 |
Result = DAG.getNode(ISD::SMAX, dl, PromotedType, Result, SatMin); |
0 |
955 |
Result = DAG.getNode(ISD::SMAX, dl, PromotedType, Result, SatMin); |
0 |
| 956 |
return Result; |
0 |
956 |
return Result; |
0 |
| 957 |
} |
0 |
957 |
} |
0 |
| 958 |
|
--- |
958 |
|
--- |
| 959 |
SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) { |
0 |
959 |
SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) { |
0 |
| 960 |
// Can just promote the operands then continue with operation. |
--- |
960 |
// Can just promote the operands then continue with operation. |
--- |
| 961 |
SDLoc dl(N); |
0 |
961 |
SDLoc dl(N); |
0 |
| 962 |
SDValue Op1Promoted, Op2Promoted; |
0 |
962 |
SDValue Op1Promoted, Op2Promoted; |
0 |
| 963 |
bool Signed = |
--- |
963 |
bool Signed = |
--- |
| 964 |
N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::SMULFIXSAT; |
0 |
964 |
N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::SMULFIXSAT; |
0 |
| 965 |
bool Saturating = |
--- |
965 |
bool Saturating = |
--- |
| 966 |
N->getOpcode() == ISD::SMULFIXSAT || N->getOpcode() == ISD::UMULFIXSAT; |
0 |
966 |
N->getOpcode() == ISD::SMULFIXSAT || N->getOpcode() == ISD::UMULFIXSAT; |
0 |
| 967 |
if (Signed) { |
0 |
967 |
if (Signed) { |
0 |
| 968 |
Op1Promoted = SExtPromotedInteger(N->getOperand(0)); |
0 |
968 |
Op1Promoted = SExtPromotedInteger(N->getOperand(0)); |
0 |
| 969 |
Op2Promoted = SExtPromotedInteger(N->getOperand(1)); |
0 |
969 |
Op2Promoted = SExtPromotedInteger(N->getOperand(1)); |
0 |
| 970 |
} else { |
--- |
970 |
} else { |
--- |
| 971 |
Op1Promoted = ZExtPromotedInteger(N->getOperand(0)); |
0 |
971 |
Op1Promoted = ZExtPromotedInteger(N->getOperand(0)); |
0 |
| 972 |
Op2Promoted = ZExtPromotedInteger(N->getOperand(1)); |
0 |
972 |
Op2Promoted = ZExtPromotedInteger(N->getOperand(1)); |
0 |
| 973 |
} |
--- |
973 |
} |
--- |
| 974 |
EVT OldType = N->getOperand(0).getValueType(); |
0 |
974 |
EVT OldType = N->getOperand(0).getValueType(); |
0 |
| 975 |
EVT PromotedType = Op1Promoted.getValueType(); |
0 |
975 |
EVT PromotedType = Op1Promoted.getValueType(); |
0 |
| 976 |
unsigned DiffSize = |
--- |
976 |
unsigned DiffSize = |
--- |
| 977 |
PromotedType.getScalarSizeInBits() - OldType.getScalarSizeInBits(); |
0 |
977 |
PromotedType.getScalarSizeInBits() - OldType.getScalarSizeInBits(); |
0 |
| 978 |
|
--- |
978 |
|
--- |
| 979 |
if (Saturating) { |
0 |
979 |
if (Saturating) { |
0 |
| 980 |
// Promoting the operand and result values changes the saturation width, |
--- |
980 |
// Promoting the operand and result values changes the saturation width, |
--- |
| 981 |
// which is extends the values that we clamp to on saturation. This could be |
--- |
981 |
// which is extends the values that we clamp to on saturation. This could be |
--- |
| 982 |
// resolved by shifting one of the operands the same amount, which would |
--- |
982 |
// resolved by shifting one of the operands the same amount, which would |
--- |
| 983 |
// also shift the result we compare against, then shifting back. |
--- |
983 |
// also shift the result we compare against, then shifting back. |
--- |
| 984 |
Op1Promoted = |
0 |
984 |
Op1Promoted = |
0 |
| 985 |
DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, |
0 |
985 |
DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, |
0 |
| 986 |
DAG.getShiftAmountConstant(DiffSize, PromotedType, dl)); |
0 |
986 |
DAG.getShiftAmountConstant(DiffSize, PromotedType, dl)); |
0 |
| 987 |
SDValue Result = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, |
0 |
987 |
SDValue Result = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, |
0 |
| 988 |
Op2Promoted, N->getOperand(2)); |
0 |
988 |
Op2Promoted, N->getOperand(2)); |
0 |
| 989 |
unsigned ShiftOp = Signed ? ISD::SRA : ISD::SRL; |
0 |
989 |
unsigned ShiftOp = Signed ? ISD::SRA : ISD::SRL; |
0 |
| 990 |
return DAG.getNode(ShiftOp, dl, PromotedType, Result, |
0 |
990 |
return DAG.getNode(ShiftOp, dl, PromotedType, Result, |
0 |
| 991 |
DAG.getShiftAmountConstant(DiffSize, PromotedType, dl)); |
0 |
991 |
DAG.getShiftAmountConstant(DiffSize, PromotedType, dl)); |
0 |
| 992 |
} |
--- |
992 |
} |
--- |
| 993 |
return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted, |
0 |
993 |
return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted, |
0 |
| 994 |
N->getOperand(2)); |
0 |
994 |
N->getOperand(2)); |
0 |
| 995 |
} |
0 |
995 |
} |
0 |
| 996 |
|
--- |
996 |
|
--- |
| 997 |
static SDValue SaturateWidenedDIVFIX(SDValue V, SDLoc &dl, |
0 |
997 |
static SDValue SaturateWidenedDIVFIX(SDValue V, SDLoc &dl, |
0 |
| 998 |
unsigned SatW, bool Signed, |
--- |
998 |
unsigned SatW, bool Signed, |
--- |
| 999 |
const TargetLowering &TLI, |
--- |
999 |
const TargetLowering &TLI, |
--- |
| 1000 |
SelectionDAG &DAG) { |
--- |
1000 |
SelectionDAG &DAG) { |
--- |
| 1001 |
EVT VT = V.getValueType(); |
0 |
1001 |
EVT VT = V.getValueType(); |
0 |
| 1002 |
unsigned VTW = VT.getScalarSizeInBits(); |
0 |
1002 |
unsigned VTW = VT.getScalarSizeInBits(); |
0 |
| 1003 |
|
--- |
1003 |
|
--- |
| 1004 |
if (!Signed) { |
0 |
1004 |
if (!Signed) { |
0 |
| 1005 |
// Saturate to the unsigned maximum by getting the minimum of V and the |
--- |
1005 |
// Saturate to the unsigned maximum by getting the minimum of V and the |
--- |
| 1006 |
// maximum. |
--- |
1006 |
// maximum. |
--- |
| 1007 |
return DAG.getNode(ISD::UMIN, dl, VT, V, |
0 |
1007 |
return DAG.getNode(ISD::UMIN, dl, VT, V, |
0 |
| 1008 |
DAG.getConstant(APInt::getLowBitsSet(VTW, SatW), |
0 |
1008 |
DAG.getConstant(APInt::getLowBitsSet(VTW, SatW), |
0 |
| 1009 |
dl, VT)); |
0 |
1009 |
dl, VT)); |
0 |
| 1010 |
} |
--- |
1010 |
} |
--- |
| 1011 |
|
--- |
1011 |
|
--- |
| 1012 |
// Saturate to the signed maximum (the low SatW - 1 bits) by taking the |
--- |
1012 |
// Saturate to the signed maximum (the low SatW - 1 bits) by taking the |
--- |
| 1013 |
// signed minimum of it and V. |
--- |
1013 |
// signed minimum of it and V. |
--- |
| 1014 |
V = DAG.getNode(ISD::SMIN, dl, VT, V, |
0 |
1014 |
V = DAG.getNode(ISD::SMIN, dl, VT, V, |
0 |
| 1015 |
DAG.getConstant(APInt::getLowBitsSet(VTW, SatW - 1), |
0 |
1015 |
DAG.getConstant(APInt::getLowBitsSet(VTW, SatW - 1), |
0 |
| 1016 |
dl, VT)); |
--- |
1016 |
dl, VT)); |
--- |
| 1017 |
// Saturate to the signed minimum (the high SatW + 1 bits) by taking the |
--- |
1017 |
// Saturate to the signed minimum (the high SatW + 1 bits) by taking the |
--- |
| 1018 |
// signed maximum of it and V. |
--- |
1018 |
// signed maximum of it and V. |
--- |
| 1019 |
V = DAG.getNode(ISD::SMAX, dl, VT, V, |
0 |
1019 |
V = DAG.getNode(ISD::SMAX, dl, VT, V, |
0 |
| 1020 |
DAG.getConstant(APInt::getHighBitsSet(VTW, VTW - SatW + 1), |
0 |
1020 |
DAG.getConstant(APInt::getHighBitsSet(VTW, VTW - SatW + 1), |
0 |
| 1021 |
dl, VT)); |
--- |
1021 |
dl, VT)); |
--- |
| 1022 |
return V; |
0 |
1022 |
return V; |
0 |
| 1023 |
} |
--- |
1023 |
} |
--- |
| 1024 |
|
--- |
1024 |
|
--- |
| 1025 |
static SDValue earlyExpandDIVFIX(SDNode *N, SDValue LHS, SDValue RHS, |
0 |
1025 |
static SDValue earlyExpandDIVFIX(SDNode *N, SDValue LHS, SDValue RHS, |
0 |
| 1026 |
unsigned Scale, const TargetLowering &TLI, |
--- |
1026 |
unsigned Scale, const TargetLowering &TLI, |
--- |
| 1027 |
SelectionDAG &DAG, unsigned SatW = 0) { |
--- |
1027 |
SelectionDAG &DAG, unsigned SatW = 0) { |
--- |
| 1028 |
EVT VT = LHS.getValueType(); |
0 |
1028 |
EVT VT = LHS.getValueType(); |
0 |
| 1029 |
unsigned VTSize = VT.getScalarSizeInBits(); |
0 |
1029 |
unsigned VTSize = VT.getScalarSizeInBits(); |
0 |
| 1030 |
bool Signed = N->getOpcode() == ISD::SDIVFIX || |
0 |
1030 |
bool Signed = N->getOpcode() == ISD::SDIVFIX || |
0 |
| 1031 |
N->getOpcode() == ISD::SDIVFIXSAT; |
0 |
1031 |
N->getOpcode() == ISD::SDIVFIXSAT; |
0 |
| 1032 |
bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT || |
0 |
1032 |
bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT || |
0 |
| 1033 |
N->getOpcode() == ISD::UDIVFIXSAT; |
0 |
1033 |
N->getOpcode() == ISD::UDIVFIXSAT; |
0 |
| 1034 |
|
--- |
1034 |
|
--- |
| 1035 |
SDLoc dl(N); |
0 |
1035 |
SDLoc dl(N); |
0 |
| 1036 |
// Widen the types by a factor of two. This is guaranteed to expand, since it |
--- |
1036 |
// Widen the types by a factor of two. This is guaranteed to expand, since it |
--- |
| 1037 |
// will always have enough high bits in the LHS to shift into. |
--- |
1037 |
// will always have enough high bits in the LHS to shift into. |
--- |
| 1038 |
EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VTSize * 2); |
0 |
1038 |
EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VTSize * 2); |
0 |
| 1039 |
if (VT.isVector()) |
0 |
1039 |
if (VT.isVector()) |
0 |
| 1040 |
WideVT = EVT::getVectorVT(*DAG.getContext(), WideVT, |
0 |
1040 |
WideVT = EVT::getVectorVT(*DAG.getContext(), WideVT, |
0 |
| 1041 |
VT.getVectorElementCount()); |
--- |
1041 |
VT.getVectorElementCount()); |
--- |
| 1042 |
LHS = DAG.getExtOrTrunc(Signed, LHS, dl, WideVT); |
0 |
1042 |
LHS = DAG.getExtOrTrunc(Signed, LHS, dl, WideVT); |
0 |
| 1043 |
RHS = DAG.getExtOrTrunc(Signed, RHS, dl, WideVT); |
0 |
1043 |
RHS = DAG.getExtOrTrunc(Signed, RHS, dl, WideVT); |
0 |
| 1044 |
SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, LHS, RHS, Scale, |
0 |
1044 |
SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, LHS, RHS, Scale, |
0 |
| 1045 |
DAG); |
--- |
1045 |
DAG); |
--- |
| 1046 |
assert(Res && "Expanding DIVFIX with wide type failed?"); |
0 |
1046 |
assert(Res && "Expanding DIVFIX with wide type failed?"); |
0 |
| 1047 |
if (Saturating) { |
0 |
1047 |
if (Saturating) { |
0 |
| 1048 |
// If the caller has told us to saturate at something less, use that width |
--- |
1048 |
// If the caller has told us to saturate at something less, use that width |
--- |
| 1049 |
// instead of the type before doubling. However, it cannot be more than |
--- |
1049 |
// instead of the type before doubling. However, it cannot be more than |
--- |
| 1050 |
// what we just widened! |
--- |
1050 |
// what we just widened! |
--- |
| 1051 |
assert(SatW <= VTSize && |
0 |
1051 |
assert(SatW <= VTSize && |
0 |
| 1052 |
"Tried to saturate to more than the original type?"); |
--- |
1052 |
"Tried to saturate to more than the original type?"); |
--- |
| 1053 |
Res = SaturateWidenedDIVFIX(Res, dl, SatW == 0 ? VTSize : SatW, Signed, |
0 |
1053 |
Res = SaturateWidenedDIVFIX(Res, dl, SatW == 0 ? VTSize : SatW, Signed, |
0 |
| 1054 |
TLI, DAG); |
--- |
1054 |
TLI, DAG); |
--- |
| 1055 |
} |
--- |
1055 |
} |
--- |
| 1056 |
return DAG.getZExtOrTrunc(Res, dl, VT); |
0 |
1056 |
return DAG.getZExtOrTrunc(Res, dl, VT); |
0 |
| 1057 |
} |
0 |
1057 |
} |
0 |
| 1058 |
|
--- |
1058 |
|
--- |
| 1059 |
SDValue DAGTypeLegalizer::PromoteIntRes_DIVFIX(SDNode *N) { |
0 |
1059 |
SDValue DAGTypeLegalizer::PromoteIntRes_DIVFIX(SDNode *N) { |
0 |
| 1060 |
SDLoc dl(N); |
0 |
1060 |
SDLoc dl(N); |
0 |
| 1061 |
SDValue Op1Promoted, Op2Promoted; |
0 |
1061 |
SDValue Op1Promoted, Op2Promoted; |
0 |
| 1062 |
bool Signed = N->getOpcode() == ISD::SDIVFIX || |
0 |
1062 |
bool Signed = N->getOpcode() == ISD::SDIVFIX || |
0 |
| 1063 |
N->getOpcode() == ISD::SDIVFIXSAT; |
0 |
1063 |
N->getOpcode() == ISD::SDIVFIXSAT; |
0 |
| 1064 |
bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT || |
0 |
1064 |
bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT || |
0 |
| 1065 |
N->getOpcode() == ISD::UDIVFIXSAT; |
0 |
1065 |
N->getOpcode() == ISD::UDIVFIXSAT; |
0 |
| 1066 |
if (Signed) { |
0 |
1066 |
if (Signed) { |
0 |
| 1067 |
Op1Promoted = SExtPromotedInteger(N->getOperand(0)); |
0 |
1067 |
Op1Promoted = SExtPromotedInteger(N->getOperand(0)); |
0 |
| 1068 |
Op2Promoted = SExtPromotedInteger(N->getOperand(1)); |
0 |
1068 |
Op2Promoted = SExtPromotedInteger(N->getOperand(1)); |
0 |
| 1069 |
} else { |
--- |
1069 |
} else { |
--- |
| 1070 |
Op1Promoted = ZExtPromotedInteger(N->getOperand(0)); |
0 |
1070 |
Op1Promoted = ZExtPromotedInteger(N->getOperand(0)); |
0 |
| 1071 |
Op2Promoted = ZExtPromotedInteger(N->getOperand(1)); |
0 |
1071 |
Op2Promoted = ZExtPromotedInteger(N->getOperand(1)); |
0 |
| 1072 |
} |
--- |
1072 |
} |
--- |
| 1073 |
EVT PromotedType = Op1Promoted.getValueType(); |
0 |
1073 |
EVT PromotedType = Op1Promoted.getValueType(); |
0 |
| 1074 |
unsigned Scale = N->getConstantOperandVal(2); |
0 |
1074 |
unsigned Scale = N->getConstantOperandVal(2); |
0 |
| 1075 |
|
--- |
1075 |
|
--- |
| 1076 |
// If the type is already legal and the operation is legal in that type, we |
--- |
1076 |
// If the type is already legal and the operation is legal in that type, we |
--- |
| 1077 |
// should not early expand. |
--- |
1077 |
// should not early expand. |
--- |
| 1078 |
if (TLI.isTypeLegal(PromotedType)) { |
0 |
1078 |
if (TLI.isTypeLegal(PromotedType)) { |
0 |
| 1079 |
TargetLowering::LegalizeAction Action = |
--- |
1079 |
TargetLowering::LegalizeAction Action = |
--- |
| 1080 |
TLI.getFixedPointOperationAction(N->getOpcode(), PromotedType, Scale); |
0 |
1080 |
TLI.getFixedPointOperationAction(N->getOpcode(), PromotedType, Scale); |
0 |
| 1081 |
if (Action == TargetLowering::Legal || Action == TargetLowering::Custom) { |
0 |
1081 |
if (Action == TargetLowering::Legal || Action == TargetLowering::Custom) { |
0 |
| 1082 |
unsigned Diff = PromotedType.getScalarSizeInBits() - |
0 |
1082 |
unsigned Diff = PromotedType.getScalarSizeInBits() - |
0 |
| 1083 |
N->getValueType(0).getScalarSizeInBits(); |
0 |
1083 |
N->getValueType(0).getScalarSizeInBits(); |
0 |
| 1084 |
if (Saturating) |
0 |
1084 |
if (Saturating) |
0 |
| 1085 |
Op1Promoted = |
0 |
1085 |
Op1Promoted = |
0 |
| 1086 |
DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, |
0 |
1086 |
DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, |
0 |
| 1087 |
DAG.getShiftAmountConstant(Diff, PromotedType, dl)); |
0 |
1087 |
DAG.getShiftAmountConstant(Diff, PromotedType, dl)); |
0 |
| 1088 |
SDValue Res = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, |
0 |
1088 |
SDValue Res = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, |
0 |
| 1089 |
Op2Promoted, N->getOperand(2)); |
0 |
1089 |
Op2Promoted, N->getOperand(2)); |
0 |
| 1090 |
if (Saturating) |
0 |
1090 |
if (Saturating) |
0 |
| 1091 |
Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, PromotedType, Res, |
0 |
1091 |
Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, PromotedType, Res, |
0 |
| 1092 |
DAG.getShiftAmountConstant(Diff, PromotedType, dl)); |
0 |
1092 |
DAG.getShiftAmountConstant(Diff, PromotedType, dl)); |
0 |
| 1093 |
return Res; |
0 |
1093 |
return Res; |
0 |
| 1094 |
} |
--- |
1094 |
} |
--- |
| 1095 |
} |
--- |
1095 |
} |
--- |
| 1096 |
|
--- |
1096 |
|
--- |
| 1097 |
// See if we can perform the division in this type without expanding. |
--- |
1097 |
// See if we can perform the division in this type without expanding. |
--- |
| 1098 |
if (SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, Op1Promoted, |
0 |
1098 |
if (SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, Op1Promoted, |
0 |
| 1099 |
Op2Promoted, Scale, DAG)) { |
0 |
1099 |
Op2Promoted, Scale, DAG)) { |
0 |
| 1100 |
if (Saturating) |
0 |
1100 |
if (Saturating) |
0 |
| 1101 |
Res = SaturateWidenedDIVFIX(Res, dl, |
0 |
1101 |
Res = SaturateWidenedDIVFIX(Res, dl, |
0 |
| 1102 |
N->getValueType(0).getScalarSizeInBits(), |
0 |
1102 |
N->getValueType(0).getScalarSizeInBits(), |
0 |
| 1103 |
Signed, TLI, DAG); |
--- |
1103 |
Signed, TLI, DAG); |
--- |
| 1104 |
return Res; |
0 |
1104 |
return Res; |
0 |
| 1105 |
} |
--- |
1105 |
} |
--- |
| 1106 |
// If we cannot, expand it to twice the type width. If we are saturating, give |
--- |
1106 |
// If we cannot, expand it to twice the type width. If we are saturating, give |
--- |
| 1107 |
// it the original width as a saturating width so we don't need to emit |
--- |
1107 |
// it the original width as a saturating width so we don't need to emit |
--- |
| 1108 |
// two saturations. |
--- |
1108 |
// two saturations. |
--- |
| 1109 |
return earlyExpandDIVFIX(N, Op1Promoted, Op2Promoted, Scale, TLI, DAG, |
0 |
1109 |
return earlyExpandDIVFIX(N, Op1Promoted, Op2Promoted, Scale, TLI, DAG, |
0 |
| 1110 |
N->getValueType(0).getScalarSizeInBits()); |
0 |
1110 |
N->getValueType(0).getScalarSizeInBits()); |
0 |
| 1111 |
} |
0 |
1111 |
} |
0 |
| 1112 |
|
--- |
1112 |
|
--- |
| 1113 |
SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) { |
0 |
1113 |
SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) { |
0 |
| 1114 |
if (ResNo == 1) |
0 |
1114 |
if (ResNo == 1) |
0 |
| 1115 |
return PromoteIntRes_Overflow(N); |
0 |
1115 |
return PromoteIntRes_Overflow(N); |
0 |
| 1116 |
|
--- |
1116 |
|
--- |
| 1117 |
// The operation overflowed iff the result in the larger type is not the |
--- |
1117 |
// The operation overflowed iff the result in the larger type is not the |
--- |
| 1118 |
// sign extension of its truncation to the original type. |
--- |
1118 |
// sign extension of its truncation to the original type. |
--- |
| 1119 |
SDValue LHS = SExtPromotedInteger(N->getOperand(0)); |
0 |
1119 |
SDValue LHS = SExtPromotedInteger(N->getOperand(0)); |
0 |
| 1120 |
SDValue RHS = SExtPromotedInteger(N->getOperand(1)); |
0 |
1120 |
SDValue RHS = SExtPromotedInteger(N->getOperand(1)); |
0 |
| 1121 |
EVT OVT = N->getOperand(0).getValueType(); |
0 |
1121 |
EVT OVT = N->getOperand(0).getValueType(); |
0 |
| 1122 |
EVT NVT = LHS.getValueType(); |
0 |
1122 |
EVT NVT = LHS.getValueType(); |
0 |
| 1123 |
SDLoc dl(N); |
0 |
1123 |
SDLoc dl(N); |
0 |
| 1124 |
|
--- |
1124 |
|
--- |
| 1125 |
// Do the arithmetic in the larger type. |
--- |
1125 |
// Do the arithmetic in the larger type. |
--- |
| 1126 |
unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB; |
0 |
1126 |
unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB; |
0 |
| 1127 |
SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); |
0 |
1127 |
SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); |
0 |
| 1128 |
|
--- |
1128 |
|
--- |
| 1129 |
// Calculate the overflow flag: sign extend the arithmetic result from |
--- |
1129 |
// Calculate the overflow flag: sign extend the arithmetic result from |
--- |
| 1130 |
// the original type. |
--- |
1130 |
// the original type. |
--- |
| 1131 |
SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, |
0 |
1131 |
SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, |
0 |
| 1132 |
DAG.getValueType(OVT)); |
0 |
1132 |
DAG.getValueType(OVT)); |
0 |
| 1133 |
// Overflowed if and only if this is not equal to Res. |
--- |
1133 |
// Overflowed if and only if this is not equal to Res. |
--- |
| 1134 |
Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); |
0 |
1134 |
Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); |
0 |
| 1135 |
|
--- |
1135 |
|
--- |
| 1136 |
// Use the calculated overflow everywhere. |
--- |
1136 |
// Use the calculated overflow everywhere. |
--- |
| 1137 |
ReplaceValueWith(SDValue(N, 1), Ofl); |
0 |
1137 |
ReplaceValueWith(SDValue(N, 1), Ofl); |
0 |
| 1138 |
|
--- |
1138 |
|
--- |
| 1139 |
return Res; |
0 |
1139 |
return Res; |
0 |
| 1140 |
} |
0 |
1140 |
} |
0 |
| 1141 |
|
--- |
1141 |
|
--- |
| 1142 |
SDValue DAGTypeLegalizer::PromoteIntRes_Select(SDNode *N) { |
0 |
1142 |
SDValue DAGTypeLegalizer::PromoteIntRes_Select(SDNode *N) { |
0 |
| 1143 |
SDValue Mask = N->getOperand(0); |
0 |
1143 |
SDValue Mask = N->getOperand(0); |
0 |
| 1144 |
|
--- |
1144 |
|
--- |
| 1145 |
SDValue LHS = GetPromotedInteger(N->getOperand(1)); |
0 |
1145 |
SDValue LHS = GetPromotedInteger(N->getOperand(1)); |
0 |
| 1146 |
SDValue RHS = GetPromotedInteger(N->getOperand(2)); |
0 |
1146 |
SDValue RHS = GetPromotedInteger(N->getOperand(2)); |
0 |
| 1147 |
|
--- |
1147 |
|
--- |
| 1148 |
unsigned Opcode = N->getOpcode(); |
0 |
1148 |
unsigned Opcode = N->getOpcode(); |
0 |
| 1149 |
if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE) |
0 |
1149 |
if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE) |
0 |
| 1150 |
return DAG.getNode(Opcode, SDLoc(N), LHS.getValueType(), Mask, LHS, RHS, |
0 |
1150 |
return DAG.getNode(Opcode, SDLoc(N), LHS.getValueType(), Mask, LHS, RHS, |
0 |
| 1151 |
N->getOperand(3)); |
0 |
1151 |
N->getOperand(3)); |
0 |
| 1152 |
return DAG.getNode(Opcode, SDLoc(N), LHS.getValueType(), Mask, LHS, RHS); |
0 |
1152 |
return DAG.getNode(Opcode, SDLoc(N), LHS.getValueType(), Mask, LHS, RHS); |
0 |
| 1153 |
} |
--- |
1153 |
} |
--- |
| 1154 |
|
--- |
1154 |
|
--- |
| 1155 |
SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) { |
0 |
1155 |
SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) { |
0 |
| 1156 |
SDValue LHS = GetPromotedInteger(N->getOperand(2)); |
0 |
1156 |
SDValue LHS = GetPromotedInteger(N->getOperand(2)); |
0 |
| 1157 |
SDValue RHS = GetPromotedInteger(N->getOperand(3)); |
0 |
1157 |
SDValue RHS = GetPromotedInteger(N->getOperand(3)); |
0 |
| 1158 |
return DAG.getNode(ISD::SELECT_CC, SDLoc(N), |
0 |
1158 |
return DAG.getNode(ISD::SELECT_CC, SDLoc(N), |
0 |
| 1159 |
LHS.getValueType(), N->getOperand(0), |
0 |
1159 |
LHS.getValueType(), N->getOperand(0), |
0 |
| 1160 |
N->getOperand(1), LHS, RHS, N->getOperand(4)); |
0 |
1160 |
N->getOperand(1), LHS, RHS, N->getOperand(4)); |
0 |
| 1161 |
} |
--- |
1161 |
} |
--- |
| 1162 |
|
--- |
1162 |
|
--- |
| 1163 |
SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) { |
1 |
1163 |
SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) { |
1 |
| 1164 |
unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0; |
1 |
1164 |
unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0; |
1 |
| 1165 |
EVT InVT = N->getOperand(OpNo).getValueType(); |
1 |
1165 |
EVT InVT = N->getOperand(OpNo).getValueType(); |
1 |
| 1166 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
1 |
1166 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
1 |
| 1167 |
|
--- |
1167 |
|
--- |
| 1168 |
EVT SVT = getSetCCResultType(InVT); |
1 |
1168 |
EVT SVT = getSetCCResultType(InVT); |
1 |
| 1169 |
|
--- |
1169 |
|
--- |
| 1170 |
// If we got back a type that needs to be promoted, this likely means the |
--- |
1170 |
// If we got back a type that needs to be promoted, this likely means the |
--- |
| 1171 |
// the input type also needs to be promoted. So get the promoted type for |
--- |
1171 |
// the input type also needs to be promoted. So get the promoted type for |
--- |
| 1172 |
// the input and try the query again. |
--- |
1172 |
// the input and try the query again. |
--- |
| 1173 |
if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) { |
1 |
1173 |
if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) { |
1 |
| 1174 |
if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) { |
0 |
1174 |
if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) { |
0 |
| 1175 |
InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT); |
0 |
1175 |
InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT); |
0 |
| 1176 |
SVT = getSetCCResultType(InVT); |
0 |
1176 |
SVT = getSetCCResultType(InVT); |
0 |
| 1177 |
} else { |
--- |
1177 |
} else { |
--- |
| 1178 |
// Input type isn't promoted, just use the default promoted type. |
--- |
1178 |
// Input type isn't promoted, just use the default promoted type. |
--- |
| 1179 |
SVT = NVT; |
0 |
1179 |
SVT = NVT; |
0 |
| 1180 |
} |
--- |
1180 |
} |
--- |
| 1181 |
} |
--- |
1181 |
} |
--- |
| 1182 |
|
--- |
1182 |
|
--- |
| 1183 |
SDLoc dl(N); |
1 |
1183 |
SDLoc dl(N); |
1 |
| 1184 |
assert(SVT.isVector() == N->getOperand(OpNo).getValueType().isVector() && |
1 |
1184 |
assert(SVT.isVector() == N->getOperand(OpNo).getValueType().isVector() && |
1 |
| 1185 |
"Vector compare must return a vector result!"); |
--- |
1185 |
"Vector compare must return a vector result!"); |
--- |
| 1186 |
|
--- |
1186 |
|
--- |
| 1187 |
// Get the SETCC result using the canonical SETCC type. |
--- |
1187 |
// Get the SETCC result using the canonical SETCC type. |
--- |
| 1188 |
SDValue SetCC; |
1 |
1188 |
SDValue SetCC; |
1 |
| 1189 |
if (N->isStrictFPOpcode()) { |
1 |
1189 |
if (N->isStrictFPOpcode()) { |
1 |
| 1190 |
SDVTList VTs = DAG.getVTList({SVT, MVT::Other}); |
0 |
1190 |
SDVTList VTs = DAG.getVTList({SVT, MVT::Other}); |
0 |
| 1191 |
SDValue Opers[] = {N->getOperand(0), N->getOperand(1), |
0 |
1191 |
SDValue Opers[] = {N->getOperand(0), N->getOperand(1), |
0 |
| 1192 |
N->getOperand(2), N->getOperand(3)}; |
0 |
1192 |
N->getOperand(2), N->getOperand(3)}; |
0 |
| 1193 |
SetCC = DAG.getNode(N->getOpcode(), dl, VTs, Opers, N->getFlags()); |
0 |
1193 |
SetCC = DAG.getNode(N->getOpcode(), dl, VTs, Opers, N->getFlags()); |
0 |
| 1194 |
// Legalize the chain result - switch anything that used the old chain to |
--- |
1194 |
// Legalize the chain result - switch anything that used the old chain to |
--- |
| 1195 |
// use the new one. |
--- |
1195 |
// use the new one. |
--- |
| 1196 |
ReplaceValueWith(SDValue(N, 1), SetCC.getValue(1)); |
0 |
1196 |
ReplaceValueWith(SDValue(N, 1), SetCC.getValue(1)); |
0 |
| 1197 |
} else |
--- |
1197 |
} else |
--- |
| 1198 |
SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0), |
2 |
1198 |
SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0), |
2 |
| 1199 |
N->getOperand(1), N->getOperand(2), N->getFlags()); |
1 |
1199 |
N->getOperand(1), N->getOperand(2), N->getFlags()); |
1 |
| 1200 |
|
--- |
1200 |
|
--- |
| 1201 |
// Convert to the expected type. |
--- |
1201 |
// Convert to the expected type. |
--- |
| 1202 |
return DAG.getSExtOrTrunc(SetCC, dl, NVT); |
1 |
1202 |
return DAG.getSExtOrTrunc(SetCC, dl, NVT); |
1 |
| 1203 |
} |
1 |
1203 |
} |
1 |
| 1204 |
|
--- |
1204 |
|
--- |
| 1205 |
SDValue DAGTypeLegalizer::PromoteIntRes_IS_FPCLASS(SDNode *N) { |
0 |
1205 |
SDValue DAGTypeLegalizer::PromoteIntRes_IS_FPCLASS(SDNode *N) { |
0 |
| 1206 |
SDLoc DL(N); |
0 |
1206 |
SDLoc DL(N); |
0 |
| 1207 |
SDValue Arg = N->getOperand(0); |
0 |
1207 |
SDValue Arg = N->getOperand(0); |
0 |
| 1208 |
SDValue Test = N->getOperand(1); |
0 |
1208 |
SDValue Test = N->getOperand(1); |
0 |
| 1209 |
EVT NResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
1209 |
EVT NResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
| 1210 |
return DAG.getNode(ISD::IS_FPCLASS, DL, NResVT, Arg, Test); |
0 |
1210 |
return DAG.getNode(ISD::IS_FPCLASS, DL, NResVT, Arg, Test); |
0 |
| 1211 |
} |
0 |
1211 |
} |
0 |
| 1212 |
|
--- |
1212 |
|
--- |
| 1213 |
SDValue DAGTypeLegalizer::PromoteIntRes_FFREXP(SDNode *N) { |
0 |
1213 |
SDValue DAGTypeLegalizer::PromoteIntRes_FFREXP(SDNode *N) { |
0 |
| 1214 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); |
0 |
1214 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); |
0 |
| 1215 |
EVT VT = N->getValueType(0); |
0 |
1215 |
EVT VT = N->getValueType(0); |
0 |
| 1216 |
|
--- |
1216 |
|
--- |
| 1217 |
SDLoc dl(N); |
0 |
1217 |
SDLoc dl(N); |
0 |
| 1218 |
SDValue Res = |
--- |
1218 |
SDValue Res = |
--- |
| 1219 |
DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, NVT), N->getOperand(0)); |
0 |
1219 |
DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, NVT), N->getOperand(0)); |
0 |
| 1220 |
|
--- |
1220 |
|
--- |
| 1221 |
ReplaceValueWith(SDValue(N, 0), Res); |
0 |
1221 |
ReplaceValueWith(SDValue(N, 0), Res); |
0 |
| 1222 |
return Res.getValue(1); |
0 |
1222 |
return Res.getValue(1); |
0 |
| 1223 |
} |
0 |
1223 |
} |
0 |
| 1224 |
|
--- |
1224 |
|
--- |
| 1225 |
SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) { |
0 |
1225 |
SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) { |
0 |
| 1226 |
SDValue LHS = GetPromotedInteger(N->getOperand(0)); |
0 |
1226 |
SDValue LHS = GetPromotedInteger(N->getOperand(0)); |
0 |
| 1227 |
SDValue RHS = N->getOperand(1); |
0 |
1227 |
SDValue RHS = N->getOperand(1); |
0 |
| 1228 |
if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) |
0 |
1228 |
if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) |
0 |
| 1229 |
RHS = ZExtPromotedInteger(RHS); |
0 |
1229 |
RHS = ZExtPromotedInteger(RHS); |
0 |
| 1230 |
if (N->getOpcode() != ISD::VP_SHL) |
0 |
1230 |
if (N->getOpcode() != ISD::VP_SHL) |
0 |
| 1231 |
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); |
0 |
1231 |
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); |
0 |
| 1232 |
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, |
0 |
1232 |
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, |
0 |
| 1233 |
N->getOperand(2), N->getOperand(3)); |
0 |
1233 |
N->getOperand(2), N->getOperand(3)); |
0 |
| 1234 |
} |
--- |
1234 |
} |
--- |
| 1235 |
|
--- |
1235 |
|
--- |
| 1236 |
SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) { |
0 |
1236 |
SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) { |
0 |
| 1237 |
SDValue Op = GetPromotedInteger(N->getOperand(0)); |
0 |
1237 |
SDValue Op = GetPromotedInteger(N->getOperand(0)); |
0 |
| 1238 |
return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), |
0 |
1238 |
return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), |
0 |
| 1239 |
Op.getValueType(), Op, N->getOperand(1)); |
0 |
1239 |
Op.getValueType(), Op, N->getOperand(1)); |
0 |
| 1240 |
} |
--- |
1240 |
} |
--- |
| 1241 |
|
--- |
1241 |
|
--- |
| 1242 |
SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) { |
0 |
1242 |
SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) { |
0 |
| 1243 |
// The input may have strange things in the top bits of the registers, but |
--- |
1243 |
// The input may have strange things in the top bits of the registers, but |
--- |
| 1244 |
// these operations don't care. They may have weird bits going out, but |
--- |
1244 |
// these operations don't care. They may have weird bits going out, but |
--- |
| 1245 |
// that too is okay if they are integer operations. |
--- |
1245 |
// that too is okay if they are integer operations. |
--- |
| 1246 |
SDValue LHS = GetPromotedInteger(N->getOperand(0)); |
0 |
1246 |
SDValue LHS = GetPromotedInteger(N->getOperand(0)); |
0 |
| 1247 |
SDValue RHS = GetPromotedInteger(N->getOperand(1)); |
0 |
1247 |
SDValue RHS = GetPromotedInteger(N->getOperand(1)); |
0 |
| 1248 |
if (N->getNumOperands() == 2) |
0 |
1248 |
if (N->getNumOperands() == 2) |
0 |
| 1249 |
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); |
0 |
1249 |
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); |
0 |
| 1250 |
assert(N->getNumOperands() == 4 && "Unexpected number of operands!"); |
0 |
1250 |
assert(N->getNumOperands() == 4 && "Unexpected number of operands!"); |
0 |
| 1251 |
assert(N->isVPOpcode() && "Expected VP opcode"); |
0 |
1251 |
assert(N->isVPOpcode() && "Expected VP opcode"); |
0 |
| 1252 |
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, |
0 |
1252 |
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, |
0 |
| 1253 |
N->getOperand(2), N->getOperand(3)); |
0 |
1253 |
N->getOperand(2), N->getOperand(3)); |
0 |
| 1254 |
} |
--- |
1254 |
} |
--- |
| 1255 |
|
--- |
1255 |
|
--- |
| 1256 |
SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) { |
0 |
1256 |
SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) { |
0 |
| 1257 |
// Sign extend the input. |
--- |
1257 |
// Sign extend the input. |
--- |
| 1258 |
SDValue LHS = SExtPromotedInteger(N->getOperand(0)); |
0 |
1258 |
SDValue LHS = SExtPromotedInteger(N->getOperand(0)); |
0 |
| 1259 |
SDValue RHS = SExtPromotedInteger(N->getOperand(1)); |
0 |
1259 |
SDValue RHS = SExtPromotedInteger(N->getOperand(1)); |
0 |
| 1260 |
if (N->getNumOperands() == 2) |
0 |
1260 |
if (N->getNumOperands() == 2) |
0 |
| 1261 |
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); |
0 |
1261 |
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); |
0 |
| 1262 |
assert(N->getNumOperands() == 4 && "Unexpected number of operands!"); |
0 |
1262 |
assert(N->getNumOperands() == 4 && "Unexpected number of operands!"); |
0 |
| 1263 |
assert(N->isVPOpcode() && "Expected VP opcode"); |
0 |
1263 |
assert(N->isVPOpcode() && "Expected VP opcode"); |
0 |
| 1264 |
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, |
0 |
1264 |
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, |
0 |
| 1265 |
N->getOperand(2), N->getOperand(3)); |
0 |
1265 |
N->getOperand(2), N->getOperand(3)); |
0 |
| 1266 |
} |
--- |
1266 |
} |
--- |
| 1267 |
|
--- |
1267 |
|
--- |
| 1268 |
SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) { |
0 |
1268 |
SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) { |
0 |
| 1269 |
// Zero extend the input. |
--- |
1269 |
// Zero extend the input. |
--- |
| 1270 |
SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); |
0 |
1270 |
SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); |
0 |
| 1271 |
SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); |
0 |
1271 |
SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); |
0 |
| 1272 |
if (N->getNumOperands() == 2) |
0 |
1272 |
if (N->getNumOperands() == 2) |
0 |
| 1273 |
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); |
0 |
1273 |
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); |
0 |
| 1274 |
assert(N->getNumOperands() == 4 && "Unexpected number of operands!"); |
0 |
1274 |
assert(N->getNumOperands() == 4 && "Unexpected number of operands!"); |
0 |
| 1275 |
assert(N->isVPOpcode() && "Expected VP opcode"); |
0 |
1275 |
assert(N->isVPOpcode() && "Expected VP opcode"); |
0 |
| 1276 |
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, |
0 |
1276 |
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, |
0 |
| 1277 |
N->getOperand(2), N->getOperand(3)); |
0 |
1277 |
N->getOperand(2), N->getOperand(3)); |
0 |
| 1278 |
} |
--- |
1278 |
} |
--- |
| 1279 |
|
--- |
1279 |
|
--- |
| 1280 |
SDValue DAGTypeLegalizer::PromoteIntRes_UMINUMAX(SDNode *N) { |
0 |
1280 |
SDValue DAGTypeLegalizer::PromoteIntRes_UMINUMAX(SDNode *N) { |
0 |
| 1281 |
// It doesn't matter if we sign extend or zero extend in the inputs. So do |
--- |
1281 |
// It doesn't matter if we sign extend or zero extend in the inputs. So do |
--- |
| 1282 |
// whatever is best for the target. |
--- |
1282 |
// whatever is best for the target. |
--- |
| 1283 |
SDValue LHS = SExtOrZExtPromotedInteger(N->getOperand(0)); |
0 |
1283 |
SDValue LHS = SExtOrZExtPromotedInteger(N->getOperand(0)); |
0 |
| 1284 |
SDValue RHS = SExtOrZExtPromotedInteger(N->getOperand(1)); |
0 |
1284 |
SDValue RHS = SExtOrZExtPromotedInteger(N->getOperand(1)); |
0 |
| 1285 |
return DAG.getNode(N->getOpcode(), SDLoc(N), |
0 |
1285 |
return DAG.getNode(N->getOpcode(), SDLoc(N), |
0 |
| 1286 |
LHS.getValueType(), LHS, RHS); |
0 |
1286 |
LHS.getValueType(), LHS, RHS); |
0 |
| 1287 |
} |
--- |
1287 |
} |
--- |
| 1288 |
|
--- |
1288 |
|
--- |
| 1289 |
SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) { |
0 |
1289 |
SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) { |
0 |
| 1290 |
// The input value must be properly sign extended. |
--- |
1290 |
// The input value must be properly sign extended. |
--- |
| 1291 |
SDValue LHS = SExtPromotedInteger(N->getOperand(0)); |
0 |
1291 |
SDValue LHS = SExtPromotedInteger(N->getOperand(0)); |
0 |
| 1292 |
SDValue RHS = N->getOperand(1); |
0 |
1292 |
SDValue RHS = N->getOperand(1); |
0 |
| 1293 |
if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) |
0 |
1293 |
if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) |
0 |
| 1294 |
RHS = ZExtPromotedInteger(RHS); |
0 |
1294 |
RHS = ZExtPromotedInteger(RHS); |
0 |
| 1295 |
if (N->getOpcode() != ISD::VP_ASHR) |
0 |
1295 |
if (N->getOpcode() != ISD::VP_ASHR) |
0 |
| 1296 |
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); |
0 |
1296 |
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); |
0 |
| 1297 |
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, |
0 |
1297 |
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, |
0 |
| 1298 |
N->getOperand(2), N->getOperand(3)); |
0 |
1298 |
N->getOperand(2), N->getOperand(3)); |
0 |
| 1299 |
} |
--- |
1299 |
} |
--- |
| 1300 |
|
--- |
1300 |
|
--- |
| 1301 |
SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) { |
0 |
1301 |
SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) { |
0 |
| 1302 |
// The input value must be properly zero extended. |
--- |
1302 |
// The input value must be properly zero extended. |
--- |
| 1303 |
SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); |
0 |
1303 |
SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); |
0 |
| 1304 |
SDValue RHS = N->getOperand(1); |
0 |
1304 |
SDValue RHS = N->getOperand(1); |
0 |
| 1305 |
if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) |
0 |
1305 |
if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) |
0 |
| 1306 |
RHS = ZExtPromotedInteger(RHS); |
0 |
1306 |
RHS = ZExtPromotedInteger(RHS); |
0 |
| 1307 |
if (N->getOpcode() != ISD::VP_LSHR) |
0 |
1307 |
if (N->getOpcode() != ISD::VP_LSHR) |
0 |
| 1308 |
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); |
0 |
1308 |
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS); |
0 |
| 1309 |
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, |
0 |
1309 |
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS, |
0 |
| 1310 |
N->getOperand(2), N->getOperand(3)); |
0 |
1310 |
N->getOperand(2), N->getOperand(3)); |
0 |
| 1311 |
} |
--- |
1311 |
} |
--- |
| 1312 |
|
--- |
1312 |
|
--- |
| 1313 |
SDValue DAGTypeLegalizer::PromoteIntRes_Rotate(SDNode *N) { |
0 |
1313 |
SDValue DAGTypeLegalizer::PromoteIntRes_Rotate(SDNode *N) { |
0 |
| 1314 |
// Lower the rotate to shifts and ORs which can be promoted. |
--- |
1314 |
// Lower the rotate to shifts and ORs which can be promoted. |
--- |
| 1315 |
SDValue Res = TLI.expandROT(N, true /*AllowVectorOps*/, DAG); |
0 |
1315 |
SDValue Res = TLI.expandROT(N, true /*AllowVectorOps*/, DAG); |
0 |
| 1316 |
ReplaceValueWith(SDValue(N, 0), Res); |
0 |
1316 |
ReplaceValueWith(SDValue(N, 0), Res); |
0 |
| 1317 |
return SDValue(); |
0 |
1317 |
return SDValue(); |
0 |
| 1318 |
} |
--- |
1318 |
} |
--- |
| 1319 |
|
--- |
1319 |
|
--- |
| 1320 |
SDValue DAGTypeLegalizer::PromoteIntRes_FunnelShift(SDNode *N) { |
0 |
1320 |
SDValue DAGTypeLegalizer::PromoteIntRes_FunnelShift(SDNode *N) { |
0 |
| 1321 |
SDValue Hi = GetPromotedInteger(N->getOperand(0)); |
0 |
1321 |
SDValue Hi = GetPromotedInteger(N->getOperand(0)); |
0 |
| 1322 |
SDValue Lo = GetPromotedInteger(N->getOperand(1)); |
0 |
1322 |
SDValue Lo = GetPromotedInteger(N->getOperand(1)); |
0 |
| 1323 |
SDValue Amt = N->getOperand(2); |
0 |
1323 |
SDValue Amt = N->getOperand(2); |
0 |
| 1324 |
if (getTypeAction(Amt.getValueType()) == TargetLowering::TypePromoteInteger) |
0 |
1324 |
if (getTypeAction(Amt.getValueType()) == TargetLowering::TypePromoteInteger) |
0 |
| 1325 |
Amt = ZExtPromotedInteger(Amt); |
0 |
1325 |
Amt = ZExtPromotedInteger(Amt); |
0 |
| 1326 |
EVT AmtVT = Amt.getValueType(); |
0 |
1326 |
EVT AmtVT = Amt.getValueType(); |
0 |
| 1327 |
|
--- |
1327 |
|
--- |
| 1328 |
SDLoc DL(N); |
0 |
1328 |
SDLoc DL(N); |
0 |
| 1329 |
EVT OldVT = N->getOperand(0).getValueType(); |
0 |
1329 |
EVT OldVT = N->getOperand(0).getValueType(); |
0 |
| 1330 |
EVT VT = Lo.getValueType(); |
0 |
1330 |
EVT VT = Lo.getValueType(); |
0 |
| 1331 |
unsigned Opcode = N->getOpcode(); |
0 |
1331 |
unsigned Opcode = N->getOpcode(); |
0 |
| 1332 |
bool IsFSHR = Opcode == ISD::FSHR; |
0 |
1332 |
bool IsFSHR = Opcode == ISD::FSHR; |
0 |
| 1333 |
unsigned OldBits = OldVT.getScalarSizeInBits(); |
0 |
1333 |
unsigned OldBits = OldVT.getScalarSizeInBits(); |
0 |
| 1334 |
unsigned NewBits = VT.getScalarSizeInBits(); |
0 |
1334 |
unsigned NewBits = VT.getScalarSizeInBits(); |
0 |
| 1335 |
|
--- |
1335 |
|
--- |
| 1336 |
// Amount has to be interpreted modulo the old bit width. |
--- |
1336 |
// Amount has to be interpreted modulo the old bit width. |
--- |
| 1337 |
Amt = DAG.getNode(ISD::UREM, DL, AmtVT, Amt, |
0 |
1337 |
Amt = DAG.getNode(ISD::UREM, DL, AmtVT, Amt, |
0 |
| 1338 |
DAG.getConstant(OldBits, DL, AmtVT)); |
0 |
1338 |
DAG.getConstant(OldBits, DL, AmtVT)); |
0 |
| 1339 |
|
--- |
1339 |
|
--- |
| 1340 |
// If the promoted type is twice the size (or more), then we use the |
--- |
1340 |
// If the promoted type is twice the size (or more), then we use the |
--- |
| 1341 |
// traditional funnel 'double' shift codegen. This isn't necessary if the |
--- |
1341 |
// traditional funnel 'double' shift codegen. This isn't necessary if the |
--- |
| 1342 |
// shift amount is constant. |
--- |
1342 |
// shift amount is constant. |
--- |
| 1343 |
// fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z % bw)) >> bw. |
--- |
1343 |
// fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z % bw)) >> bw. |
--- |
| 1344 |
// fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z % bw)). |
--- |
1344 |
// fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z % bw)). |
--- |
| 1345 |
if (NewBits >= (2 * OldBits) && !isa(Amt) && |
0 |
1345 |
if (NewBits >= (2 * OldBits) && !isa(Amt) && |
0 |
| 1346 |
!TLI.isOperationLegalOrCustom(Opcode, VT)) { |
0 |
1346 |
!TLI.isOperationLegalOrCustom(Opcode, VT)) { |
0 |
| 1347 |
SDValue HiShift = DAG.getConstant(OldBits, DL, VT); |
0 |
1347 |
SDValue HiShift = DAG.getConstant(OldBits, DL, VT); |
0 |
| 1348 |
Hi = DAG.getNode(ISD::SHL, DL, VT, Hi, HiShift); |
0 |
1348 |
Hi = DAG.getNode(ISD::SHL, DL, VT, Hi, HiShift); |
0 |
| 1349 |
Lo = DAG.getZeroExtendInReg(Lo, DL, OldVT); |
0 |
1349 |
Lo = DAG.getZeroExtendInReg(Lo, DL, OldVT); |
0 |
| 1350 |
SDValue Res = DAG.getNode(ISD::OR, DL, VT, Hi, Lo); |
0 |
1350 |
SDValue Res = DAG.getNode(ISD::OR, DL, VT, Hi, Lo); |
0 |
| 1351 |
Res = DAG.getNode(IsFSHR ? ISD::SRL : ISD::SHL, DL, VT, Res, Amt); |
0 |
1351 |
Res = DAG.getNode(IsFSHR ? ISD::SRL : ISD::SHL, DL, VT, Res, Amt); |
0 |
| 1352 |
if (!IsFSHR) |
0 |
1352 |
if (!IsFSHR) |
0 |
| 1353 |
Res = DAG.getNode(ISD::SRL, DL, VT, Res, HiShift); |
0 |
1353 |
Res = DAG.getNode(ISD::SRL, DL, VT, Res, HiShift); |
0 |
| 1354 |
return Res; |
0 |
1354 |
return Res; |
0 |
| 1355 |
} |
--- |
1355 |
} |
--- |
| 1356 |
|
--- |
1356 |
|
--- |
| 1357 |
// Shift Lo up to occupy the upper bits of the promoted type. |
--- |
1357 |
// Shift Lo up to occupy the upper bits of the promoted type. |
--- |
| 1358 |
SDValue ShiftOffset = DAG.getConstant(NewBits - OldBits, DL, AmtVT); |
0 |
1358 |
SDValue ShiftOffset = DAG.getConstant(NewBits - OldBits, DL, AmtVT); |
0 |
| 1359 |
Lo = DAG.getNode(ISD::SHL, DL, VT, Lo, ShiftOffset); |
0 |
1359 |
Lo = DAG.getNode(ISD::SHL, DL, VT, Lo, ShiftOffset); |
0 |
| 1360 |
|
--- |
1360 |
|
--- |
| 1361 |
// Increase Amount to shift the result into the lower bits of the promoted |
--- |
1361 |
// Increase Amount to shift the result into the lower bits of the promoted |
--- |
| 1362 |
// type. |
--- |
1362 |
// type. |
--- |
| 1363 |
if (IsFSHR) |
0 |
1363 |
if (IsFSHR) |
0 |
| 1364 |
Amt = DAG.getNode(ISD::ADD, DL, AmtVT, Amt, ShiftOffset); |
0 |
1364 |
Amt = DAG.getNode(ISD::ADD, DL, AmtVT, Amt, ShiftOffset); |
0 |
| 1365 |
|
--- |
1365 |
|
--- |
| 1366 |
return DAG.getNode(Opcode, DL, VT, Hi, Lo, Amt); |
0 |
1366 |
return DAG.getNode(Opcode, DL, VT, Hi, Lo, Amt); |
0 |
| 1367 |
} |
0 |
1367 |
} |
0 |
| 1368 |
|
--- |
1368 |
|
--- |
| 1369 |
SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) { |
0 |
1369 |
SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) { |
0 |
| 1370 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
1370 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
| 1371 |
SDValue Res; |
0 |
1371 |
SDValue Res; |
0 |
| 1372 |
SDValue InOp = N->getOperand(0); |
0 |
1372 |
SDValue InOp = N->getOperand(0); |
0 |
| 1373 |
SDLoc dl(N); |
0 |
1373 |
SDLoc dl(N); |
0 |
| 1374 |
|
--- |
1374 |
|
--- |
| 1375 |
switch (getTypeAction(InOp.getValueType())) { |
0 |
1375 |
switch (getTypeAction(InOp.getValueType())) { |
0 |
| 1376 |
default: llvm_unreachable("Unknown type action!"); |
0 |
1376 |
default: llvm_unreachable("Unknown type action!"); |
0 |
| 1377 |
case TargetLowering::TypeLegal: |
0 |
1377 |
case TargetLowering::TypeLegal: |
0 |
| 1378 |
case TargetLowering::TypeExpandInteger: |
--- |
1378 |
case TargetLowering::TypeExpandInteger: |
--- |
| 1379 |
Res = InOp; |
0 |
1379 |
Res = InOp; |
0 |
| 1380 |
break; |
0 |
1380 |
break; |
0 |
| 1381 |
case TargetLowering::TypePromoteInteger: |
0 |
1381 |
case TargetLowering::TypePromoteInteger: |
0 |
| 1382 |
Res = GetPromotedInteger(InOp); |
0 |
1382 |
Res = GetPromotedInteger(InOp); |
0 |
| 1383 |
break; |
0 |
1383 |
break; |
0 |
| 1384 |
case TargetLowering::TypeSplitVector: { |
0 |
1384 |
case TargetLowering::TypeSplitVector: { |
0 |
| 1385 |
EVT InVT = InOp.getValueType(); |
0 |
1385 |
EVT InVT = InOp.getValueType(); |
0 |
| 1386 |
assert(InVT.isVector() && "Cannot split scalar types"); |
0 |
1386 |
assert(InVT.isVector() && "Cannot split scalar types"); |
0 |
| 1387 |
ElementCount NumElts = InVT.getVectorElementCount(); |
0 |
1387 |
ElementCount NumElts = InVT.getVectorElementCount(); |
0 |
| 1388 |
assert(NumElts == NVT.getVectorElementCount() && |
0 |
1388 |
assert(NumElts == NVT.getVectorElementCount() && |
0 |
| 1389 |
"Dst and Src must have the same number of elements"); |
--- |
1389 |
"Dst and Src must have the same number of elements"); |
--- |
| 1390 |
assert(isPowerOf2_32(NumElts.getKnownMinValue()) && |
0 |
1390 |
assert(isPowerOf2_32(NumElts.getKnownMinValue()) && |
0 |
| 1391 |
"Promoted vector type must be a power of two"); |
--- |
1391 |
"Promoted vector type must be a power of two"); |
--- |
| 1392 |
|
--- |
1392 |
|
--- |
| 1393 |
SDValue EOp1, EOp2; |
0 |
1393 |
SDValue EOp1, EOp2; |
0 |
| 1394 |
GetSplitVector(InOp, EOp1, EOp2); |
0 |
1394 |
GetSplitVector(InOp, EOp1, EOp2); |
0 |
| 1395 |
|
--- |
1395 |
|
--- |
| 1396 |
EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(), |
0 |
1396 |
EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(), |
0 |
| 1397 |
NumElts.divideCoefficientBy(2)); |
--- |
1397 |
NumElts.divideCoefficientBy(2)); |
--- |
| 1398 |
if (N->getOpcode() == ISD::TRUNCATE) { |
0 |
1398 |
if (N->getOpcode() == ISD::TRUNCATE) { |
0 |
| 1399 |
EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1); |
0 |
1399 |
EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1); |
0 |
| 1400 |
EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2); |
0 |
1400 |
EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2); |
0 |
| 1401 |
} else { |
--- |
1401 |
} else { |
--- |
| 1402 |
assert(N->getOpcode() == ISD::VP_TRUNCATE && |
0 |
1402 |
assert(N->getOpcode() == ISD::VP_TRUNCATE && |
0 |
| 1403 |
"Expected VP_TRUNCATE opcode"); |
--- |
1403 |
"Expected VP_TRUNCATE opcode"); |
--- |
| 1404 |
SDValue MaskLo, MaskHi, EVLLo, EVLHi; |
0 |
1404 |
SDValue MaskLo, MaskHi, EVLLo, EVLHi; |
0 |
| 1405 |
std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1)); |
0 |
1405 |
std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1)); |
0 |
| 1406 |
std::tie(EVLLo, EVLHi) = |
0 |
1406 |
std::tie(EVLLo, EVLHi) = |
0 |
| 1407 |
DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl); |
0 |
1407 |
DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl); |
0 |
| 1408 |
EOp1 = DAG.getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp1, MaskLo, EVLLo); |
0 |
1408 |
EOp1 = DAG.getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp1, MaskLo, EVLLo); |
0 |
| 1409 |
EOp2 = DAG.getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp2, MaskHi, EVLHi); |
0 |
1409 |
EOp2 = DAG.getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp2, MaskHi, EVLHi); |
0 |
| 1410 |
} |
--- |
1410 |
} |
--- |
| 1411 |
return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2); |
0 |
1411 |
return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2); |
0 |
| 1412 |
} |
--- |
1412 |
} |
--- |
| 1413 |
// TODO: VP_TRUNCATE need to handle when TypeWidenVector access to some |
--- |
1413 |
// TODO: VP_TRUNCATE need to handle when TypeWidenVector access to some |
--- |
| 1414 |
// targets. |
--- |
1414 |
// targets. |
--- |
| 1415 |
case TargetLowering::TypeWidenVector: { |
0 |
1415 |
case TargetLowering::TypeWidenVector: { |
0 |
| 1416 |
SDValue WideInOp = GetWidenedVector(InOp); |
0 |
1416 |
SDValue WideInOp = GetWidenedVector(InOp); |
0 |
| 1417 |
|
--- |
1417 |
|
--- |
| 1418 |
// Truncate widened InOp. |
--- |
1418 |
// Truncate widened InOp. |
--- |
| 1419 |
unsigned NumElem = WideInOp.getValueType().getVectorNumElements(); |
0 |
1419 |
unsigned NumElem = WideInOp.getValueType().getVectorNumElements(); |
0 |
| 1420 |
EVT TruncVT = EVT::getVectorVT(*DAG.getContext(), |
0 |
1420 |
EVT TruncVT = EVT::getVectorVT(*DAG.getContext(), |
0 |
| 1421 |
N->getValueType(0).getScalarType(), NumElem); |
0 |
1421 |
N->getValueType(0).getScalarType(), NumElem); |
0 |
| 1422 |
SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp); |
0 |
1422 |
SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp); |
0 |
| 1423 |
|
--- |
1423 |
|
--- |
| 1424 |
// Zero extend so that the elements are of same type as those of NVT |
--- |
1424 |
// Zero extend so that the elements are of same type as those of NVT |
--- |
| 1425 |
EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(), |
0 |
1425 |
EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(), |
0 |
| 1426 |
NumElem); |
--- |
1426 |
NumElem); |
--- |
| 1427 |
SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc); |
0 |
1427 |
SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc); |
0 |
| 1428 |
|
--- |
1428 |
|
--- |
| 1429 |
// Extract the low NVT subvector. |
--- |
1429 |
// Extract the low NVT subvector. |
--- |
| 1430 |
SDValue ZeroIdx = DAG.getVectorIdxConstant(0, dl); |
0 |
1430 |
SDValue ZeroIdx = DAG.getVectorIdxConstant(0, dl); |
0 |
| 1431 |
return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx); |
0 |
1431 |
return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx); |
0 |
| 1432 |
} |
--- |
1432 |
} |
--- |
| 1433 |
} |
--- |
1433 |
} |
--- |
| 1434 |
|
--- |
1434 |
|
--- |
| 1435 |
// Truncate to NVT instead of VT |
--- |
1435 |
// Truncate to NVT instead of VT |
--- |
| 1436 |
if (N->getOpcode() == ISD::VP_TRUNCATE) |
0 |
1436 |
if (N->getOpcode() == ISD::VP_TRUNCATE) |
0 |
| 1437 |
return DAG.getNode(ISD::VP_TRUNCATE, dl, NVT, Res, N->getOperand(1), |
0 |
1437 |
return DAG.getNode(ISD::VP_TRUNCATE, dl, NVT, Res, N->getOperand(1), |
0 |
| 1438 |
N->getOperand(2)); |
0 |
1438 |
N->getOperand(2)); |
0 |
| 1439 |
return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res); |
0 |
1439 |
return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res); |
0 |
| 1440 |
} |
0 |
1440 |
} |
0 |
| 1441 |
|
--- |
1441 |
|
--- |
| 1442 |
SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) { |
0 |
1442 |
SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) { |
0 |
| 1443 |
if (ResNo == 1) |
0 |
1443 |
if (ResNo == 1) |
0 |
| 1444 |
return PromoteIntRes_Overflow(N); |
0 |
1444 |
return PromoteIntRes_Overflow(N); |
0 |
| 1445 |
|
--- |
1445 |
|
--- |
| 1446 |
// The operation overflowed iff the result in the larger type is not the |
--- |
1446 |
// The operation overflowed iff the result in the larger type is not the |
--- |
| 1447 |
// zero extension of its truncation to the original type. |
--- |
1447 |
// zero extension of its truncation to the original type. |
--- |
| 1448 |
SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); |
0 |
1448 |
SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); |
0 |
| 1449 |
SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); |
0 |
1449 |
SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); |
0 |
| 1450 |
EVT OVT = N->getOperand(0).getValueType(); |
0 |
1450 |
EVT OVT = N->getOperand(0).getValueType(); |
0 |
| 1451 |
EVT NVT = LHS.getValueType(); |
0 |
1451 |
EVT NVT = LHS.getValueType(); |
0 |
| 1452 |
SDLoc dl(N); |
0 |
1452 |
SDLoc dl(N); |
0 |
| 1453 |
|
--- |
1453 |
|
--- |
| 1454 |
// Do the arithmetic in the larger type. |
--- |
1454 |
// Do the arithmetic in the larger type. |
--- |
| 1455 |
unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB; |
0 |
1455 |
unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB; |
0 |
| 1456 |
SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); |
0 |
1456 |
SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); |
0 |
| 1457 |
|
--- |
1457 |
|
--- |
| 1458 |
// Calculate the overflow flag: zero extend the arithmetic result from |
--- |
1458 |
// Calculate the overflow flag: zero extend the arithmetic result from |
--- |
| 1459 |
// the original type. |
--- |
1459 |
// the original type. |
--- |
| 1460 |
SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT); |
0 |
1460 |
SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT); |
0 |
| 1461 |
// Overflowed if and only if this is not equal to Res. |
--- |
1461 |
// Overflowed if and only if this is not equal to Res. |
--- |
| 1462 |
Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); |
0 |
1462 |
Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); |
0 |
| 1463 |
|
--- |
1463 |
|
--- |
| 1464 |
// Use the calculated overflow everywhere. |
--- |
1464 |
// Use the calculated overflow everywhere. |
--- |
| 1465 |
ReplaceValueWith(SDValue(N, 1), Ofl); |
0 |
1465 |
ReplaceValueWith(SDValue(N, 1), Ofl); |
0 |
| 1466 |
|
--- |
1466 |
|
--- |
| 1467 |
return Res; |
0 |
1467 |
return Res; |
0 |
| 1468 |
} |
0 |
1468 |
} |
0 |
| 1469 |
|
--- |
1469 |
|
--- |
| 1470 |
// Handle promotion for the ADDE/SUBE/UADDO_CARRY/USUBO_CARRY nodes. Notice that |
--- |
1470 |
// Handle promotion for the ADDE/SUBE/UADDO_CARRY/USUBO_CARRY nodes. Notice that |
--- |
| 1471 |
// the third operand of ADDE/SUBE nodes is carry flag, which differs from |
--- |
1471 |
// the third operand of ADDE/SUBE nodes is carry flag, which differs from |
--- |
| 1472 |
// the UADDO_CARRY/USUBO_CARRY nodes in that the third operand is carry Boolean. |
--- |
1472 |
// the UADDO_CARRY/USUBO_CARRY nodes in that the third operand is carry Boolean. |
--- |
| 1473 |
SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO_CARRY(SDNode *N, |
0 |
1473 |
SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO_CARRY(SDNode *N, |
0 |
| 1474 |
unsigned ResNo) { |
--- |
1474 |
unsigned ResNo) { |
--- |
| 1475 |
if (ResNo == 1) |
0 |
1475 |
if (ResNo == 1) |
0 |
| 1476 |
return PromoteIntRes_Overflow(N); |
0 |
1476 |
return PromoteIntRes_Overflow(N); |
0 |
| 1477 |
|
--- |
1477 |
|
--- |
| 1478 |
// We need to sign-extend the operands so the carry value computed by the |
--- |
1478 |
// We need to sign-extend the operands so the carry value computed by the |
--- |
| 1479 |
// wide operation will be equivalent to the carry value computed by the |
--- |
1479 |
// wide operation will be equivalent to the carry value computed by the |
--- |
| 1480 |
// narrow operation. |
--- |
1480 |
// narrow operation. |
--- |
| 1481 |
// An UADDO_CARRY can generate carry only if any of the operands has its |
--- |
1481 |
// An UADDO_CARRY can generate carry only if any of the operands has its |
--- |
| 1482 |
// most significant bit set. Sign extension propagates the most significant |
--- |
1482 |
// most significant bit set. Sign extension propagates the most significant |
--- |
| 1483 |
// bit into the higher bits which means the extra bit that the narrow |
--- |
1483 |
// bit into the higher bits which means the extra bit that the narrow |
--- |
| 1484 |
// addition would need (i.e. the carry) will be propagated through the higher |
--- |
1484 |
// addition would need (i.e. the carry) will be propagated through the higher |
--- |
| 1485 |
// bits of the wide addition. |
--- |
1485 |
// bits of the wide addition. |
--- |
| 1486 |
// A USUBO_CARRY can generate borrow only if LHS < RHS and this property will |
--- |
1486 |
// A USUBO_CARRY can generate borrow only if LHS < RHS and this property will |
--- |
| 1487 |
// be preserved by sign extension. |
--- |
1487 |
// be preserved by sign extension. |
--- |
| 1488 |
SDValue LHS = SExtPromotedInteger(N->getOperand(0)); |
0 |
1488 |
SDValue LHS = SExtPromotedInteger(N->getOperand(0)); |
0 |
| 1489 |
SDValue RHS = SExtPromotedInteger(N->getOperand(1)); |
0 |
1489 |
SDValue RHS = SExtPromotedInteger(N->getOperand(1)); |
0 |
| 1490 |
|
--- |
1490 |
|
--- |
| 1491 |
EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)}; |
0 |
1491 |
EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)}; |
0 |
| 1492 |
|
--- |
1492 |
|
--- |
| 1493 |
// Do the arithmetic in the wide type. |
--- |
1493 |
// Do the arithmetic in the wide type. |
--- |
| 1494 |
SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs), |
0 |
1494 |
SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs), |
0 |
| 1495 |
LHS, RHS, N->getOperand(2)); |
0 |
1495 |
LHS, RHS, N->getOperand(2)); |
0 |
| 1496 |
|
--- |
1496 |
|
--- |
| 1497 |
// Update the users of the original carry/borrow value. |
--- |
1497 |
// Update the users of the original carry/borrow value. |
--- |
| 1498 |
ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); |
0 |
1498 |
ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); |
0 |
| 1499 |
|
--- |
1499 |
|
--- |
| 1500 |
return SDValue(Res.getNode(), 0); |
0 |
1500 |
return SDValue(Res.getNode(), 0); |
0 |
| 1501 |
} |
--- |
1501 |
} |
--- |
| 1502 |
|
--- |
1502 |
|
--- |
| 1503 |
SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO_CARRY(SDNode *N, |
0 |
1503 |
SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO_CARRY(SDNode *N, |
0 |
| 1504 |
unsigned ResNo) { |
--- |
1504 |
unsigned ResNo) { |
--- |
| 1505 |
assert(ResNo == 1 && "Don't know how to promote other results yet."); |
0 |
1505 |
assert(ResNo == 1 && "Don't know how to promote other results yet."); |
0 |
| 1506 |
return PromoteIntRes_Overflow(N); |
0 |
1506 |
return PromoteIntRes_Overflow(N); |
0 |
| 1507 |
} |
--- |
1507 |
} |
--- |
| 1508 |
|
--- |
1508 |
|
--- |
| 1509 |
SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) { |
0 |
1509 |
SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) { |
0 |
| 1510 |
EVT OVT = N->getValueType(0); |
0 |
1510 |
EVT OVT = N->getValueType(0); |
0 |
| 1511 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); |
0 |
1511 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); |
0 |
| 1512 |
|
--- |
1512 |
|
--- |
| 1513 |
// If a larger ABS or SMAX isn't supported by the target, try to expand now. |
--- |
1513 |
// If a larger ABS or SMAX isn't supported by the target, try to expand now. |
--- |
| 1514 |
// If we expand later we'll end up sign extending more than just the sra input |
--- |
1514 |
// If we expand later we'll end up sign extending more than just the sra input |
--- |
| 1515 |
// in sra+xor+sub expansion. |
--- |
1515 |
// in sra+xor+sub expansion. |
--- |
| 1516 |
if (!OVT.isVector() && |
0 |
1516 |
if (!OVT.isVector() && |
0 |
| 1517 |
!TLI.isOperationLegalOrCustomOrPromote(ISD::ABS, NVT) && |
0 |
1517 |
!TLI.isOperationLegalOrCustomOrPromote(ISD::ABS, NVT) && |
0 |
| 1518 |
!TLI.isOperationLegal(ISD::SMAX, NVT)) { |
0 |
1518 |
!TLI.isOperationLegal(ISD::SMAX, NVT)) { |
0 |
| 1519 |
if (SDValue Res = TLI.expandABS(N, DAG)) |
0 |
1519 |
if (SDValue Res = TLI.expandABS(N, DAG)) |
0 |
| 1520 |
return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Res); |
0 |
1520 |
return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Res); |
0 |
| 1521 |
} |
--- |
1521 |
} |
--- |
| 1522 |
|
--- |
1522 |
|
--- |
| 1523 |
SDValue Op0 = SExtPromotedInteger(N->getOperand(0)); |
0 |
1523 |
SDValue Op0 = SExtPromotedInteger(N->getOperand(0)); |
0 |
| 1524 |
return DAG.getNode(ISD::ABS, SDLoc(N), Op0.getValueType(), Op0); |
0 |
1524 |
return DAG.getNode(ISD::ABS, SDLoc(N), Op0.getValueType(), Op0); |
0 |
| 1525 |
} |
--- |
1525 |
} |
--- |
| 1526 |
|
--- |
1526 |
|
--- |
| 1527 |
SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) { |
0 |
1527 |
SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) { |
0 |
| 1528 |
// Promote the overflow bit trivially. |
--- |
1528 |
// Promote the overflow bit trivially. |
--- |
| 1529 |
if (ResNo == 1) |
0 |
1529 |
if (ResNo == 1) |
0 |
| 1530 |
return PromoteIntRes_Overflow(N); |
0 |
1530 |
return PromoteIntRes_Overflow(N); |
0 |
| 1531 |
|
--- |
1531 |
|
--- |
| 1532 |
SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); |
0 |
1532 |
SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); |
0 |
| 1533 |
SDLoc DL(N); |
0 |
1533 |
SDLoc DL(N); |
0 |
| 1534 |
EVT SmallVT = LHS.getValueType(); |
0 |
1534 |
EVT SmallVT = LHS.getValueType(); |
0 |
| 1535 |
|
--- |
1535 |
|
--- |
| 1536 |
// To determine if the result overflowed in a larger type, we extend the |
--- |
1536 |
// To determine if the result overflowed in a larger type, we extend the |
--- |
| 1537 |
// input to the larger type, do the multiply (checking if it overflows), |
--- |
1537 |
// input to the larger type, do the multiply (checking if it overflows), |
--- |
| 1538 |
// then also check the high bits of the result to see if overflow happened |
--- |
1538 |
// then also check the high bits of the result to see if overflow happened |
--- |
| 1539 |
// there. |
--- |
1539 |
// there. |
--- |
| 1540 |
if (N->getOpcode() == ISD::SMULO) { |
0 |
1540 |
if (N->getOpcode() == ISD::SMULO) { |
0 |
| 1541 |
LHS = SExtPromotedInteger(LHS); |
0 |
1541 |
LHS = SExtPromotedInteger(LHS); |
0 |
| 1542 |
RHS = SExtPromotedInteger(RHS); |
0 |
1542 |
RHS = SExtPromotedInteger(RHS); |
0 |
| 1543 |
} else { |
--- |
1543 |
} else { |
--- |
| 1544 |
LHS = ZExtPromotedInteger(LHS); |
0 |
1544 |
LHS = ZExtPromotedInteger(LHS); |
0 |
| 1545 |
RHS = ZExtPromotedInteger(RHS); |
0 |
1545 |
RHS = ZExtPromotedInteger(RHS); |
0 |
| 1546 |
} |
--- |
1546 |
} |
--- |
| 1547 |
SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1)); |
0 |
1547 |
SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1)); |
0 |
| 1548 |
SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS); |
0 |
1548 |
SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS); |
0 |
| 1549 |
|
--- |
1549 |
|
--- |
| 1550 |
// Overflow occurred if it occurred in the larger type, or if the high part |
--- |
1550 |
// Overflow occurred if it occurred in the larger type, or if the high part |
--- |
| 1551 |
// of the result does not zero/sign-extend the low part. Check this second |
--- |
1551 |
// of the result does not zero/sign-extend the low part. Check this second |
--- |
| 1552 |
// possibility first. |
--- |
1552 |
// possibility first. |
--- |
| 1553 |
SDValue Overflow; |
0 |
1553 |
SDValue Overflow; |
0 |
| 1554 |
if (N->getOpcode() == ISD::UMULO) { |
0 |
1554 |
if (N->getOpcode() == ISD::UMULO) { |
0 |
| 1555 |
// Unsigned overflow occurred if the high part is non-zero. |
--- |
1555 |
// Unsigned overflow occurred if the high part is non-zero. |
--- |
| 1556 |
unsigned Shift = SmallVT.getScalarSizeInBits(); |
0 |
1556 |
unsigned Shift = SmallVT.getScalarSizeInBits(); |
0 |
| 1557 |
SDValue Hi = |
--- |
1557 |
SDValue Hi = |
--- |
| 1558 |
DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul, |
0 |
1558 |
DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul, |
0 |
| 1559 |
DAG.getShiftAmountConstant(Shift, Mul.getValueType(), DL)); |
0 |
1559 |
DAG.getShiftAmountConstant(Shift, Mul.getValueType(), DL)); |
0 |
| 1560 |
Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi, |
0 |
1560 |
Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi, |
0 |
| 1561 |
DAG.getConstant(0, DL, Hi.getValueType()), |
0 |
1561 |
DAG.getConstant(0, DL, Hi.getValueType()), |
0 |
| 1562 |
ISD::SETNE); |
--- |
1562 |
ISD::SETNE); |
--- |
| 1563 |
} else { |
--- |
1563 |
} else { |
--- |
| 1564 |
// Signed overflow occurred if the high part does not sign extend the low. |
--- |
1564 |
// Signed overflow occurred if the high part does not sign extend the low. |
--- |
| 1565 |
SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(), |
0 |
1565 |
SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(), |
0 |
| 1566 |
Mul, DAG.getValueType(SmallVT)); |
0 |
1566 |
Mul, DAG.getValueType(SmallVT)); |
0 |
| 1567 |
Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE); |
0 |
1567 |
Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE); |
0 |
| 1568 |
} |
--- |
1568 |
} |
--- |
| 1569 |
|
--- |
1569 |
|
--- |
| 1570 |
// The only other way for overflow to occur is if the multiplication in the |
--- |
1570 |
// The only other way for overflow to occur is if the multiplication in the |
--- |
| 1571 |
// larger type itself overflowed. |
--- |
1571 |
// larger type itself overflowed. |
--- |
| 1572 |
Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow, |
0 |
1572 |
Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow, |
0 |
| 1573 |
SDValue(Mul.getNode(), 1)); |
--- |
1573 |
SDValue(Mul.getNode(), 1)); |
--- |
| 1574 |
|
--- |
1574 |
|
--- |
| 1575 |
// Use the calculated overflow everywhere. |
--- |
1575 |
// Use the calculated overflow everywhere. |
--- |
| 1576 |
ReplaceValueWith(SDValue(N, 1), Overflow); |
0 |
1576 |
ReplaceValueWith(SDValue(N, 1), Overflow); |
0 |
| 1577 |
return Mul; |
0 |
1577 |
return Mul; |
0 |
| 1578 |
} |
0 |
1578 |
} |
0 |
| 1579 |
|
--- |
1579 |
|
--- |
| 1580 |
SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) { |
0 |
1580 |
SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) { |
0 |
| 1581 |
return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), |
0 |
1581 |
return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), |
0 |
| 1582 |
N->getValueType(0))); |
0 |
1582 |
N->getValueType(0))); |
0 |
| 1583 |
} |
--- |
1583 |
} |
--- |
| 1584 |
|
--- |
1584 |
|
--- |
| 1585 |
SDValue DAGTypeLegalizer::PromoteIntRes_VSCALE(SDNode *N) { |
0 |
1585 |
SDValue DAGTypeLegalizer::PromoteIntRes_VSCALE(SDNode *N) { |
0 |
| 1586 |
EVT VT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
1586 |
EVT VT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
| 1587 |
|
--- |
1587 |
|
--- |
| 1588 |
const APInt &MulImm = N->getConstantOperandAPInt(0); |
0 |
1588 |
const APInt &MulImm = N->getConstantOperandAPInt(0); |
0 |
| 1589 |
return DAG.getVScale(SDLoc(N), VT, MulImm.sext(VT.getSizeInBits())); |
0 |
1589 |
return DAG.getVScale(SDLoc(N), VT, MulImm.sext(VT.getSizeInBits())); |
0 |
| 1590 |
} |
--- |
1590 |
} |
--- |
| 1591 |
|
--- |
1591 |
|
--- |
| 1592 |
SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) { |
0 |
1592 |
SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) { |
0 |
| 1593 |
SDValue Chain = N->getOperand(0); // Get the chain. |
0 |
1593 |
SDValue Chain = N->getOperand(0); // Get the chain. |
0 |
| 1594 |
SDValue Ptr = N->getOperand(1); // Get the pointer. |
0 |
1594 |
SDValue Ptr = N->getOperand(1); // Get the pointer. |
0 |
| 1595 |
EVT VT = N->getValueType(0); |
0 |
1595 |
EVT VT = N->getValueType(0); |
0 |
| 1596 |
SDLoc dl(N); |
0 |
1596 |
SDLoc dl(N); |
0 |
| 1597 |
|
--- |
1597 |
|
--- |
| 1598 |
MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT); |
0 |
1598 |
MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT); |
0 |
| 1599 |
unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT); |
0 |
1599 |
unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT); |
0 |
| 1600 |
// The argument is passed as NumRegs registers of type RegVT. |
--- |
1600 |
// The argument is passed as NumRegs registers of type RegVT. |
--- |
| 1601 |
|
--- |
1601 |
|
--- |
| 1602 |
SmallVector Parts(NumRegs); |
0 |
1602 |
SmallVector Parts(NumRegs); |
0 |
| 1603 |
for (unsigned i = 0; i < NumRegs; ++i) { |
0 |
1603 |
for (unsigned i = 0; i < NumRegs; ++i) { |
0 |
| 1604 |
Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2), |
0 |
1604 |
Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2), |
0 |
| 1605 |
N->getConstantOperandVal(3)); |
0 |
1605 |
N->getConstantOperandVal(3)); |
0 |
| 1606 |
Chain = Parts[i].getValue(1); |
0 |
1606 |
Chain = Parts[i].getValue(1); |
0 |
| 1607 |
} |
--- |
1607 |
} |
--- |
| 1608 |
|
--- |
1608 |
|
--- |
| 1609 |
// Handle endianness of the load. |
--- |
1609 |
// Handle endianness of the load. |
--- |
| 1610 |
if (DAG.getDataLayout().isBigEndian()) |
0 |
1610 |
if (DAG.getDataLayout().isBigEndian()) |
0 |
| 1611 |
std::reverse(Parts.begin(), Parts.end()); |
0 |
1611 |
std::reverse(Parts.begin(), Parts.end()); |
0 |
| 1612 |
|
--- |
1612 |
|
--- |
| 1613 |
// Assemble the parts in the promoted type. |
--- |
1613 |
// Assemble the parts in the promoted type. |
--- |
| 1614 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
1614 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
| 1615 |
SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]); |
0 |
1615 |
SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]); |
0 |
| 1616 |
for (unsigned i = 1; i < NumRegs; ++i) { |
0 |
1616 |
for (unsigned i = 1; i < NumRegs; ++i) { |
0 |
| 1617 |
SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]); |
0 |
1617 |
SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]); |
0 |
| 1618 |
// Shift it to the right position and "or" it in. |
--- |
1618 |
// Shift it to the right position and "or" it in. |
--- |
| 1619 |
Part = DAG.getNode(ISD::SHL, dl, NVT, Part, |
0 |
1619 |
Part = DAG.getNode(ISD::SHL, dl, NVT, Part, |
0 |
| 1620 |
DAG.getConstant(i * RegVT.getSizeInBits(), dl, |
0 |
1620 |
DAG.getConstant(i * RegVT.getSizeInBits(), dl, |
0 |
| 1621 |
TLI.getPointerTy(DAG.getDataLayout()))); |
0 |
1621 |
TLI.getPointerTy(DAG.getDataLayout()))); |
0 |
| 1622 |
Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part); |
0 |
1622 |
Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part); |
0 |
| 1623 |
} |
--- |
1623 |
} |
--- |
| 1624 |
|
--- |
1624 |
|
--- |
| 1625 |
// Modified the chain result - switch anything that used the old chain to |
--- |
1625 |
// Modified the chain result - switch anything that used the old chain to |
--- |
| 1626 |
// use the new one. |
--- |
1626 |
// use the new one. |
--- |
| 1627 |
ReplaceValueWith(SDValue(N, 1), Chain); |
0 |
1627 |
ReplaceValueWith(SDValue(N, 1), Chain); |
0 |
| 1628 |
|
--- |
1628 |
|
--- |
| 1629 |
return Res; |
0 |
1629 |
return Res; |
0 |
| 1630 |
} |
0 |
1630 |
} |
0 |
| 1631 |
|
--- |
1631 |
|
--- |
| 1632 |
//===----------------------------------------------------------------------===// |
--- |
1632 |
//===----------------------------------------------------------------------===// |
--- |
| 1633 |
// Integer Operand Promotion |
--- |
1633 |
// Integer Operand Promotion |
--- |
| 1634 |
//===----------------------------------------------------------------------===// |
--- |
1634 |
//===----------------------------------------------------------------------===// |
--- |
| 1635 |
|
--- |
1635 |
|
--- |
| 1636 |
/// PromoteIntegerOperand - This method is called when the specified operand of |
--- |
1636 |
/// PromoteIntegerOperand - This method is called when the specified operand of |
--- |
| 1637 |
/// the specified node is found to need promotion. At this point, all of the |
--- |
1637 |
/// the specified node is found to need promotion. At this point, all of the |
--- |
| 1638 |
/// result types of the node are known to be legal, but other operands of the |
--- |
1638 |
/// result types of the node are known to be legal, but other operands of the |
--- |
| 1639 |
/// node may need promotion or expansion as well as the specified one. |
--- |
1639 |
/// node may need promotion or expansion as well as the specified one. |
--- |
| 1640 |
bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) { |
2 |
1640 |
bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) { |
2 |
| 1641 |
LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG); |
2 |
1641 |
LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG); |
2 |
| 1642 |
dbgs() << "\n"); |
--- |
1642 |
dbgs() << "\n"); |
--- |
| 1643 |
SDValue Res = SDValue(); |
2 |
1643 |
SDValue Res = SDValue(); |
2 |
| 1644 |
if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) { |
2 |
1644 |
if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) { |
2 |
| 1645 |
LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n"); |
0 |
1645 |
LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n"); |
0 |
| 1646 |
return false; |
0 |
1646 |
return false; |
0 |
| 1647 |
} |
--- |
1647 |
} |
--- |
| 1648 |
|
--- |
1648 |
|
--- |
| 1649 |
switch (N->getOpcode()) { |
2 |
1649 |
switch (N->getOpcode()) { |
2 |
| 1650 |
default: |
0 |
1650 |
default: |
0 |
| 1651 |
#ifndef NDEBUG |
--- |
1651 |
#ifndef NDEBUG |
--- |
| 1652 |
dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": "; |
0 |
1652 |
dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": "; |
0 |
| 1653 |
N->dump(&DAG); dbgs() << "\n"; |
0 |
1653 |
N->dump(&DAG); dbgs() << "\n"; |
0 |
| 1654 |
#endif |
--- |
1654 |
#endif |
--- |
| 1655 |
report_fatal_error("Do not know how to promote this operator's operand!"); |
0 |
1655 |
report_fatal_error("Do not know how to promote this operator's operand!"); |
0 |
| 1656 |
|
--- |
1656 |
|
--- |
| 1657 |
case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break; |
0 |
1657 |
case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break; |
0 |
| 1658 |
case ISD::ATOMIC_STORE: |
0 |
1658 |
case ISD::ATOMIC_STORE: |
0 |
| 1659 |
Res = PromoteIntOp_ATOMIC_STORE(cast(N)); |
0 |
1659 |
Res = PromoteIntOp_ATOMIC_STORE(cast(N)); |
0 |
| 1660 |
break; |
0 |
1660 |
break; |
0 |
| 1661 |
case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break; |
0 |
1661 |
case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break; |
0 |
| 1662 |
case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break; |
0 |
1662 |
case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break; |
0 |
| 1663 |
case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break; |
1 |
1663 |
case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break; |
1 |
| 1664 |
case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break; |
0 |
1664 |
case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break; |
0 |
| 1665 |
case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break; |
0 |
1665 |
case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break; |
0 |
| 1666 |
case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break; |
0 |
1666 |
case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break; |
0 |
| 1667 |
case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break; |
0 |
1667 |
case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break; |
0 |
| 1668 |
case ISD::INSERT_VECTOR_ELT: |
0 |
1668 |
case ISD::INSERT_VECTOR_ELT: |
0 |
| 1669 |
Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo); |
0 |
1669 |
Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo); |
0 |
| 1670 |
break; |
0 |
1670 |
break; |
0 |
| 1671 |
case ISD::SPLAT_VECTOR: |
0 |
1671 |
case ISD::SPLAT_VECTOR: |
0 |
| 1672 |
case ISD::SCALAR_TO_VECTOR: |
--- |
1672 |
case ISD::SCALAR_TO_VECTOR: |
--- |
| 1673 |
Res = PromoteIntOp_ScalarOp(N); |
0 |
1673 |
Res = PromoteIntOp_ScalarOp(N); |
0 |
| 1674 |
break; |
0 |
1674 |
break; |
0 |
| 1675 |
case ISD::VSELECT: |
0 |
1675 |
case ISD::VSELECT: |
0 |
| 1676 |
case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break; |
0 |
1676 |
case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break; |
0 |
| 1677 |
case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break; |
0 |
1677 |
case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break; |
0 |
| 1678 |
case ISD::VP_SETCC: |
0 |
1678 |
case ISD::VP_SETCC: |
0 |
| 1679 |
case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break; |
0 |
1679 |
case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break; |
0 |
| 1680 |
case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break; |
0 |
1680 |
case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break; |
0 |
| 1681 |
case ISD::VP_SIGN_EXTEND: Res = PromoteIntOp_VP_SIGN_EXTEND(N); break; |
0 |
1681 |
case ISD::VP_SIGN_EXTEND: Res = PromoteIntOp_VP_SIGN_EXTEND(N); break; |
0 |
| 1682 |
case ISD::VP_SINT_TO_FP: |
0 |
1682 |
case ISD::VP_SINT_TO_FP: |
0 |
| 1683 |
case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break; |
0 |
1683 |
case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break; |
0 |
| 1684 |
case ISD::STRICT_SINT_TO_FP: Res = PromoteIntOp_STRICT_SINT_TO_FP(N); break; |
0 |
1684 |
case ISD::STRICT_SINT_TO_FP: Res = PromoteIntOp_STRICT_SINT_TO_FP(N); break; |
0 |
| 1685 |
case ISD::STORE: Res = PromoteIntOp_STORE(cast(N), |
0 |
1685 |
case ISD::STORE: Res = PromoteIntOp_STORE(cast(N), |
0 |
| 1686 |
OpNo); break; |
0 |
1686 |
OpNo); break; |
0 |
| 1687 |
case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast(N), |
0 |
1687 |
case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast(N), |
0 |
| 1688 |
OpNo); break; |
0 |
1688 |
OpNo); break; |
0 |
| 1689 |
case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast(N), |
0 |
1689 |
case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast(N), |
0 |
| 1690 |
OpNo); break; |
0 |
1690 |
OpNo); break; |
0 |
| 1691 |
case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast(N), |
0 |
1691 |
case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast(N), |
0 |
| 1692 |
OpNo); break; |
0 |
1692 |
OpNo); break; |
0 |
| 1693 |
case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast(N), |
0 |
1693 |
case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast(N), |
0 |
| 1694 |
OpNo); break; |
0 |
1694 |
OpNo); break; |
0 |
| 1695 |
case ISD::VP_TRUNCATE: |
0 |
1695 |
case ISD::VP_TRUNCATE: |
0 |
| 1696 |
case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break; |
0 |
1696 |
case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break; |
0 |
| 1697 |
case ISD::BF16_TO_FP: |
0 |
1697 |
case ISD::BF16_TO_FP: |
0 |
| 1698 |
case ISD::FP16_TO_FP: |
--- |
1698 |
case ISD::FP16_TO_FP: |
--- |
| 1699 |
case ISD::VP_UINT_TO_FP: |
--- |
1699 |
case ISD::VP_UINT_TO_FP: |
--- |
| 1700 |
case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break; |
0 |
1700 |
case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break; |
0 |
| 1701 |
case ISD::STRICT_UINT_TO_FP: Res = PromoteIntOp_STRICT_UINT_TO_FP(N); break; |
0 |
1701 |
case ISD::STRICT_UINT_TO_FP: Res = PromoteIntOp_STRICT_UINT_TO_FP(N); break; |
0 |
| 1702 |
case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break; |
1 |
1702 |
case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break; |
1 |
| 1703 |
case ISD::VP_ZERO_EXTEND: Res = PromoteIntOp_VP_ZERO_EXTEND(N); break; |
0 |
1703 |
case ISD::VP_ZERO_EXTEND: Res = PromoteIntOp_VP_ZERO_EXTEND(N); break; |
0 |
| 1704 |
case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break; |
0 |
1704 |
case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break; |
0 |
| 1705 |
case ISD::INSERT_SUBVECTOR: Res = PromoteIntOp_INSERT_SUBVECTOR(N); break; |
0 |
1705 |
case ISD::INSERT_SUBVECTOR: Res = PromoteIntOp_INSERT_SUBVECTOR(N); break; |
0 |
| 1706 |
|
--- |
1706 |
|
--- |
| 1707 |
case ISD::SHL: |
0 |
1707 |
case ISD::SHL: |
0 |
| 1708 |
case ISD::SRA: |
--- |
1708 |
case ISD::SRA: |
--- |
| 1709 |
case ISD::SRL: |
--- |
1709 |
case ISD::SRL: |
--- |
| 1710 |
case ISD::ROTL: |
--- |
1710 |
case ISD::ROTL: |
--- |
| 1711 |
case ISD::ROTR: Res = PromoteIntOp_Shift(N); break; |
0 |
1711 |
case ISD::ROTR: Res = PromoteIntOp_Shift(N); break; |
0 |
| 1712 |
|
--- |
1712 |
|
--- |
| 1713 |
case ISD::FSHL: |
0 |
1713 |
case ISD::FSHL: |
0 |
| 1714 |
case ISD::FSHR: Res = PromoteIntOp_FunnelShift(N); break; |
0 |
1714 |
case ISD::FSHR: Res = PromoteIntOp_FunnelShift(N); break; |
0 |
| 1715 |
|
--- |
1715 |
|
--- |
| 1716 |
case ISD::SADDO_CARRY: |
0 |
1716 |
case ISD::SADDO_CARRY: |
0 |
| 1717 |
case ISD::SSUBO_CARRY: |
--- |
1717 |
case ISD::SSUBO_CARRY: |
--- |
| 1718 |
case ISD::UADDO_CARRY: |
--- |
1718 |
case ISD::UADDO_CARRY: |
--- |
| 1719 |
case ISD::USUBO_CARRY: Res = PromoteIntOp_ADDSUBO_CARRY(N, OpNo); break; |
0 |
1719 |
case ISD::USUBO_CARRY: Res = PromoteIntOp_ADDSUBO_CARRY(N, OpNo); break; |
0 |
| 1720 |
|
--- |
1720 |
|
--- |
| 1721 |
case ISD::FRAMEADDR: |
0 |
1721 |
case ISD::FRAMEADDR: |
0 |
| 1722 |
case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break; |
0 |
1722 |
case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break; |
0 |
| 1723 |
|
--- |
1723 |
|
--- |
| 1724 |
case ISD::PREFETCH: Res = PromoteIntOp_PREFETCH(N, OpNo); break; |
0 |
1724 |
case ISD::PREFETCH: Res = PromoteIntOp_PREFETCH(N, OpNo); break; |
0 |
| 1725 |
|
--- |
1725 |
|
--- |
| 1726 |
case ISD::SMULFIX: |
0 |
1726 |
case ISD::SMULFIX: |
0 |
| 1727 |
case ISD::SMULFIXSAT: |
--- |
1727 |
case ISD::SMULFIXSAT: |
--- |
| 1728 |
case ISD::UMULFIX: |
--- |
1728 |
case ISD::UMULFIX: |
--- |
| 1729 |
case ISD::UMULFIXSAT: |
--- |
1729 |
case ISD::UMULFIXSAT: |
--- |
| 1730 |
case ISD::SDIVFIX: |
--- |
1730 |
case ISD::SDIVFIX: |
--- |
| 1731 |
case ISD::SDIVFIXSAT: |
--- |
1731 |
case ISD::SDIVFIXSAT: |
--- |
| 1732 |
case ISD::UDIVFIX: |
--- |
1732 |
case ISD::UDIVFIX: |
--- |
| 1733 |
case ISD::UDIVFIXSAT: Res = PromoteIntOp_FIX(N); break; |
0 |
1733 |
case ISD::UDIVFIXSAT: Res = PromoteIntOp_FIX(N); break; |
0 |
| 1734 |
case ISD::FPOWI: |
0 |
1734 |
case ISD::FPOWI: |
0 |
| 1735 |
case ISD::STRICT_FPOWI: |
--- |
1735 |
case ISD::STRICT_FPOWI: |
--- |
| 1736 |
case ISD::FLDEXP: |
--- |
1736 |
case ISD::FLDEXP: |
--- |
| 1737 |
case ISD::STRICT_FLDEXP: Res = PromoteIntOp_ExpOp(N); break; |
0 |
1737 |
case ISD::STRICT_FLDEXP: Res = PromoteIntOp_ExpOp(N); break; |
0 |
| 1738 |
case ISD::VECREDUCE_ADD: |
0 |
1738 |
case ISD::VECREDUCE_ADD: |
0 |
| 1739 |
case ISD::VECREDUCE_MUL: |
--- |
1739 |
case ISD::VECREDUCE_MUL: |
--- |
| 1740 |
case ISD::VECREDUCE_AND: |
--- |
1740 |
case ISD::VECREDUCE_AND: |
--- |
| 1741 |
case ISD::VECREDUCE_OR: |
--- |
1741 |
case ISD::VECREDUCE_OR: |
--- |
| 1742 |
case ISD::VECREDUCE_XOR: |
--- |
1742 |
case ISD::VECREDUCE_XOR: |
--- |
| 1743 |
case ISD::VECREDUCE_SMAX: |
--- |
1743 |
case ISD::VECREDUCE_SMAX: |
--- |
| 1744 |
case ISD::VECREDUCE_SMIN: |
--- |
1744 |
case ISD::VECREDUCE_SMIN: |
--- |
| 1745 |
case ISD::VECREDUCE_UMAX: |
--- |
1745 |
case ISD::VECREDUCE_UMAX: |
--- |
| 1746 |
case ISD::VECREDUCE_UMIN: Res = PromoteIntOp_VECREDUCE(N); break; |
0 |
1746 |
case ISD::VECREDUCE_UMIN: Res = PromoteIntOp_VECREDUCE(N); break; |
0 |
| 1747 |
case ISD::VP_REDUCE_ADD: |
0 |
1747 |
case ISD::VP_REDUCE_ADD: |
0 |
| 1748 |
case ISD::VP_REDUCE_MUL: |
--- |
1748 |
case ISD::VP_REDUCE_MUL: |
--- |
| 1749 |
case ISD::VP_REDUCE_AND: |
--- |
1749 |
case ISD::VP_REDUCE_AND: |
--- |
| 1750 |
case ISD::VP_REDUCE_OR: |
--- |
1750 |
case ISD::VP_REDUCE_OR: |
--- |
| 1751 |
case ISD::VP_REDUCE_XOR: |
--- |
1751 |
case ISD::VP_REDUCE_XOR: |
--- |
| 1752 |
case ISD::VP_REDUCE_SMAX: |
--- |
1752 |
case ISD::VP_REDUCE_SMAX: |
--- |
| 1753 |
case ISD::VP_REDUCE_SMIN: |
--- |
1753 |
case ISD::VP_REDUCE_SMIN: |
--- |
| 1754 |
case ISD::VP_REDUCE_UMAX: |
--- |
1754 |
case ISD::VP_REDUCE_UMAX: |
--- |
| 1755 |
case ISD::VP_REDUCE_UMIN: |
--- |
1755 |
case ISD::VP_REDUCE_UMIN: |
--- |
| 1756 |
Res = PromoteIntOp_VP_REDUCE(N, OpNo); |
0 |
1756 |
Res = PromoteIntOp_VP_REDUCE(N, OpNo); |
0 |
| 1757 |
break; |
0 |
1757 |
break; |
0 |
| 1758 |
|
--- |
1758 |
|
--- |
| 1759 |
case ISD::SET_ROUNDING: Res = PromoteIntOp_SET_ROUNDING(N); break; |
0 |
1759 |
case ISD::SET_ROUNDING: Res = PromoteIntOp_SET_ROUNDING(N); break; |
0 |
| 1760 |
case ISD::STACKMAP: |
0 |
1760 |
case ISD::STACKMAP: |
0 |
| 1761 |
Res = PromoteIntOp_STACKMAP(N, OpNo); |
0 |
1761 |
Res = PromoteIntOp_STACKMAP(N, OpNo); |
0 |
| 1762 |
break; |
0 |
1762 |
break; |
0 |
| 1763 |
case ISD::PATCHPOINT: |
0 |
1763 |
case ISD::PATCHPOINT: |
0 |
| 1764 |
Res = PromoteIntOp_PATCHPOINT(N, OpNo); |
0 |
1764 |
Res = PromoteIntOp_PATCHPOINT(N, OpNo); |
0 |
| 1765 |
break; |
0 |
1765 |
break; |
0 |
| 1766 |
case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: |
0 |
1766 |
case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: |
0 |
| 1767 |
case ISD::EXPERIMENTAL_VP_STRIDED_STORE: |
--- |
1767 |
case ISD::EXPERIMENTAL_VP_STRIDED_STORE: |
--- |
| 1768 |
Res = PromoteIntOp_VP_STRIDED(N, OpNo); |
0 |
1768 |
Res = PromoteIntOp_VP_STRIDED(N, OpNo); |
0 |
| 1769 |
break; |
0 |
1769 |
break; |
0 |
| 1770 |
} |
--- |
1770 |
} |
--- |
| 1771 |
|
--- |
1771 |
|
--- |
| 1772 |
// If the result is null, the sub-method took care of registering results etc. |
--- |
1772 |
// If the result is null, the sub-method took care of registering results etc. |
--- |
| 1773 |
if (!Res.getNode()) return false; |
2 |
1773 |
if (!Res.getNode()) return false; |
2 |
| 1774 |
|
--- |
1774 |
|
--- |
| 1775 |
// If the result is N, the sub-method updated N in place. Tell the legalizer |
--- |
1775 |
// If the result is N, the sub-method updated N in place. Tell the legalizer |
--- |
| 1776 |
// core about this. |
--- |
1776 |
// core about this. |
--- |
| 1777 |
if (Res.getNode() == N) |
2 |
1777 |
if (Res.getNode() == N) |
2 |
| 1778 |
return true; |
1 |
1778 |
return true; |
1 |
| 1779 |
|
--- |
1779 |
|
--- |
| 1780 |
const bool IsStrictFp = N->isStrictFPOpcode(); |
1 |
1780 |
const bool IsStrictFp = N->isStrictFPOpcode(); |
1 |
| 1781 |
assert(Res.getValueType() == N->getValueType(0) && |
1 |
1781 |
assert(Res.getValueType() == N->getValueType(0) && |
1 |
| 1782 |
N->getNumValues() == (IsStrictFp ? 2 : 1) && |
--- |
1782 |
N->getNumValues() == (IsStrictFp ? 2 : 1) && |
--- |
| 1783 |
"Invalid operand expansion"); |
--- |
1783 |
"Invalid operand expansion"); |
--- |
| 1784 |
LLVM_DEBUG(dbgs() << "Replacing: "; N->dump(&DAG); dbgs() << " with: "; |
1 |
1784 |
LLVM_DEBUG(dbgs() << "Replacing: "; N->dump(&DAG); dbgs() << " with: "; |
1 |
| 1785 |
Res.dump()); |
--- |
1785 |
Res.dump()); |
--- |
| 1786 |
|
--- |
1786 |
|
--- |
| 1787 |
ReplaceValueWith(SDValue(N, 0), Res); |
1 |
1787 |
ReplaceValueWith(SDValue(N, 0), Res); |
1 |
| 1788 |
if (IsStrictFp) |
1 |
1788 |
if (IsStrictFp) |
1 |
| 1789 |
ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1)); |
0 |
1789 |
ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1)); |
0 |
| 1790 |
|
--- |
1790 |
|
--- |
| 1791 |
return false; |
1 |
1791 |
return false; |
1 |
| 1792 |
} |
--- |
1792 |
} |
--- |
| 1793 |
|
--- |
1793 |
|
--- |
| 1794 |
/// PromoteSetCCOperands - Promote the operands of a comparison. This code is |
--- |
1794 |
/// PromoteSetCCOperands - Promote the operands of a comparison. This code is |
--- |
| 1795 |
/// shared among BR_CC, SELECT_CC, and SETCC handlers. |
--- |
1795 |
/// shared among BR_CC, SELECT_CC, and SETCC handlers. |
--- |
| 1796 |
void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &LHS, SDValue &RHS, |
0 |
1796 |
void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &LHS, SDValue &RHS, |
0 |
| 1797 |
ISD::CondCode CCCode) { |
--- |
1797 |
ISD::CondCode CCCode) { |
--- |
| 1798 |
// We have to insert explicit sign or zero extends. Note that we could |
--- |
1798 |
// We have to insert explicit sign or zero extends. Note that we could |
--- |
| 1799 |
// insert sign extends for ALL conditions. For those operations where either |
--- |
1799 |
// insert sign extends for ALL conditions. For those operations where either |
--- |
| 1800 |
// zero or sign extension would be valid, we ask the target which extension |
--- |
1800 |
// zero or sign extension would be valid, we ask the target which extension |
--- |
| 1801 |
// it would prefer. |
--- |
1801 |
// it would prefer. |
--- |
| 1802 |
|
--- |
1802 |
|
--- |
| 1803 |
// Signed comparisons always require sign extension. |
--- |
1803 |
// Signed comparisons always require sign extension. |
--- |
| 1804 |
if (ISD::isSignedIntSetCC(CCCode)) { |
0 |
1804 |
if (ISD::isSignedIntSetCC(CCCode)) { |
0 |
| 1805 |
LHS = SExtPromotedInteger(LHS); |
0 |
1805 |
LHS = SExtPromotedInteger(LHS); |
0 |
| 1806 |
RHS = SExtPromotedInteger(RHS); |
0 |
1806 |
RHS = SExtPromotedInteger(RHS); |
0 |
| 1807 |
return; |
0 |
1807 |
return; |
0 |
| 1808 |
} |
--- |
1808 |
} |
--- |
| 1809 |
|
--- |
1809 |
|
--- |
| 1810 |
assert((ISD::isUnsignedIntSetCC(CCCode) || ISD::isIntEqualitySetCC(CCCode)) && |
0 |
1810 |
assert((ISD::isUnsignedIntSetCC(CCCode) || ISD::isIntEqualitySetCC(CCCode)) && |
0 |
| 1811 |
"Unknown integer comparison!"); |
--- |
1811 |
"Unknown integer comparison!"); |
--- |
| 1812 |
|
--- |
1812 |
|
--- |
| 1813 |
SDValue OpL = GetPromotedInteger(LHS); |
0 |
1813 |
SDValue OpL = GetPromotedInteger(LHS); |
0 |
| 1814 |
SDValue OpR = GetPromotedInteger(RHS); |
0 |
1814 |
SDValue OpR = GetPromotedInteger(RHS); |
0 |
| 1815 |
|
--- |
1815 |
|
--- |
| 1816 |
if (TLI.isSExtCheaperThanZExt(LHS.getValueType(), OpL.getValueType())) { |
0 |
1816 |
if (TLI.isSExtCheaperThanZExt(LHS.getValueType(), OpL.getValueType())) { |
0 |
| 1817 |
// The target would prefer to promote the comparison operand with sign |
--- |
1817 |
// The target would prefer to promote the comparison operand with sign |
--- |
| 1818 |
// extension. Honor that unless the promoted values are already zero |
--- |
1818 |
// extension. Honor that unless the promoted values are already zero |
--- |
| 1819 |
// extended. |
--- |
1819 |
// extended. |
--- |
| 1820 |
unsigned OpLEffectiveBits = |
--- |
1820 |
unsigned OpLEffectiveBits = |
--- |
| 1821 |
DAG.computeKnownBits(OpL).countMaxActiveBits(); |
0 |
1821 |
DAG.computeKnownBits(OpL).countMaxActiveBits(); |
0 |
| 1822 |
unsigned OpREffectiveBits = |
--- |
1822 |
unsigned OpREffectiveBits = |
--- |
| 1823 |
DAG.computeKnownBits(OpR).countMaxActiveBits(); |
0 |
1823 |
DAG.computeKnownBits(OpR).countMaxActiveBits(); |
0 |
| 1824 |
if (OpLEffectiveBits <= LHS.getScalarValueSizeInBits() && |
0 |
1824 |
if (OpLEffectiveBits <= LHS.getScalarValueSizeInBits() && |
0 |
| 1825 |
OpREffectiveBits <= RHS.getScalarValueSizeInBits()) { |
0 |
1825 |
OpREffectiveBits <= RHS.getScalarValueSizeInBits()) { |
0 |
| 1826 |
LHS = OpL; |
0 |
1826 |
LHS = OpL; |
0 |
| 1827 |
RHS = OpR; |
0 |
1827 |
RHS = OpR; |
0 |
| 1828 |
return; |
0 |
1828 |
return; |
0 |
| 1829 |
} |
--- |
1829 |
} |
--- |
| 1830 |
|
--- |
1830 |
|
--- |
| 1831 |
// The promoted values aren't zero extended, use a sext_inreg. |
--- |
1831 |
// The promoted values aren't zero extended, use a sext_inreg. |
--- |
| 1832 |
LHS = SExtPromotedInteger(LHS); |
0 |
1832 |
LHS = SExtPromotedInteger(LHS); |
0 |
| 1833 |
RHS = SExtPromotedInteger(RHS); |
0 |
1833 |
RHS = SExtPromotedInteger(RHS); |
0 |
| 1834 |
return; |
0 |
1834 |
return; |
0 |
| 1835 |
} |
--- |
1835 |
} |
--- |
| 1836 |
|
--- |
1836 |
|
--- |
| 1837 |
// Prefer to promote the comparison operand with zero extension. |
--- |
1837 |
// Prefer to promote the comparison operand with zero extension. |
--- |
| 1838 |
|
--- |
1838 |
|
--- |
| 1839 |
// If the width of OpL/OpR excluding the duplicated sign bits is no greater |
--- |
1839 |
// If the width of OpL/OpR excluding the duplicated sign bits is no greater |
--- |
| 1840 |
// than the width of LHS/RHS, we can avoid/ inserting a zext_inreg operation |
--- |
1840 |
// than the width of LHS/RHS, we can avoid/ inserting a zext_inreg operation |
--- |
| 1841 |
// that we might not be able to remove. |
--- |
1841 |
// that we might not be able to remove. |
--- |
| 1842 |
unsigned OpLEffectiveBits = DAG.ComputeMaxSignificantBits(OpL); |
0 |
1842 |
unsigned OpLEffectiveBits = DAG.ComputeMaxSignificantBits(OpL); |
0 |
| 1843 |
unsigned OpREffectiveBits = DAG.ComputeMaxSignificantBits(OpR); |
0 |
1843 |
unsigned OpREffectiveBits = DAG.ComputeMaxSignificantBits(OpR); |
0 |
| 1844 |
if (OpLEffectiveBits <= LHS.getScalarValueSizeInBits() && |
0 |
1844 |
if (OpLEffectiveBits <= LHS.getScalarValueSizeInBits() && |
0 |
| 1845 |
OpREffectiveBits <= RHS.getScalarValueSizeInBits()) { |
0 |
1845 |
OpREffectiveBits <= RHS.getScalarValueSizeInBits()) { |
0 |
| 1846 |
LHS = OpL; |
0 |
1846 |
LHS = OpL; |
0 |
| 1847 |
RHS = OpR; |
0 |
1847 |
RHS = OpR; |
0 |
| 1848 |
return; |
0 |
1848 |
return; |
0 |
| 1849 |
} |
--- |
1849 |
} |
--- |
| 1850 |
|
--- |
1850 |
|
--- |
| 1851 |
// Otherwise, use zext_inreg. |
--- |
1851 |
// Otherwise, use zext_inreg. |
--- |
| 1852 |
LHS = ZExtPromotedInteger(LHS); |
0 |
1852 |
LHS = ZExtPromotedInteger(LHS); |
0 |
| 1853 |
RHS = ZExtPromotedInteger(RHS); |
0 |
1853 |
RHS = ZExtPromotedInteger(RHS); |
0 |
| 1854 |
} |
--- |
1854 |
} |
--- |
| 1855 |
|
--- |
1855 |
|
--- |
| 1856 |
SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) { |
0 |
1856 |
SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) { |
0 |
| 1857 |
SDValue Op = GetPromotedInteger(N->getOperand(0)); |
0 |
1857 |
SDValue Op = GetPromotedInteger(N->getOperand(0)); |
0 |
| 1858 |
return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op); |
0 |
1858 |
return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op); |
0 |
| 1859 |
} |
--- |
1859 |
} |
--- |
| 1860 |
|
--- |
1860 |
|
--- |
| 1861 |
SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) { |
0 |
1861 |
SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) { |
0 |
| 1862 |
SDValue Op2 = GetPromotedInteger(N->getOperand(2)); |
0 |
1862 |
SDValue Op2 = GetPromotedInteger(N->getOperand(2)); |
0 |
| 1863 |
return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(), |
0 |
1863 |
return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(), |
0 |
| 1864 |
N->getChain(), N->getBasePtr(), Op2, N->getMemOperand()); |
0 |
1864 |
N->getChain(), N->getBasePtr(), Op2, N->getMemOperand()); |
0 |
| 1865 |
} |
--- |
1865 |
} |
--- |
| 1866 |
|
--- |
1866 |
|
--- |
| 1867 |
SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) { |
0 |
1867 |
SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) { |
0 |
| 1868 |
// This should only occur in unusual situations like bitcasting to an |
--- |
1868 |
// This should only occur in unusual situations like bitcasting to an |
--- |
| 1869 |
// x86_fp80, so just turn it into a store+load |
--- |
1869 |
// x86_fp80, so just turn it into a store+load |
--- |
| 1870 |
return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0)); |
0 |
1870 |
return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0)); |
0 |
| 1871 |
} |
--- |
1871 |
} |
--- |
| 1872 |
|
--- |
1872 |
|
--- |
| 1873 |
SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) { |
0 |
1873 |
SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) { |
0 |
| 1874 |
assert(OpNo == 2 && "Don't know how to promote this operand!"); |
0 |
1874 |
assert(OpNo == 2 && "Don't know how to promote this operand!"); |
0 |
| 1875 |
|
--- |
1875 |
|
--- |
| 1876 |
SDValue LHS = N->getOperand(2); |
0 |
1876 |
SDValue LHS = N->getOperand(2); |
0 |
| 1877 |
SDValue RHS = N->getOperand(3); |
0 |
1877 |
SDValue RHS = N->getOperand(3); |
0 |
| 1878 |
PromoteSetCCOperands(LHS, RHS, cast(N->getOperand(1))->get()); |
0 |
1878 |
PromoteSetCCOperands(LHS, RHS, cast(N->getOperand(1))->get()); |
0 |
| 1879 |
|
--- |
1879 |
|
--- |
| 1880 |
// The chain (Op#0), CC (#1) and basic block destination (Op#4) are always |
--- |
1880 |
// The chain (Op#0), CC (#1) and basic block destination (Op#4) are always |
--- |
| 1881 |
// legal types. |
--- |
1881 |
// legal types. |
--- |
| 1882 |
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), |
0 |
1882 |
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), |
0 |
| 1883 |
N->getOperand(1), LHS, RHS, N->getOperand(4)), |
0 |
1883 |
N->getOperand(1), LHS, RHS, N->getOperand(4)), |
0 |
| 1884 |
0); |
0 |
1884 |
0); |
0 |
| 1885 |
} |
--- |
1885 |
} |
--- |
| 1886 |
|
--- |
1886 |
|
--- |
| 1887 |
SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) { |
1 |
1887 |
SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) { |
1 |
| 1888 |
assert(OpNo == 1 && "only know how to promote condition"); |
1 |
1888 |
assert(OpNo == 1 && "only know how to promote condition"); |
1 |
| 1889 |
|
--- |
1889 |
|
--- |
| 1890 |
// Promote all the way up to the canonical SetCC type. |
--- |
1890 |
// Promote all the way up to the canonical SetCC type. |
--- |
| 1891 |
SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other); |
1 |
1891 |
SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other); |
1 |
| 1892 |
|
--- |
1892 |
|
--- |
| 1893 |
// The chain (Op#0) and basic block destination (Op#2) are always legal types. |
--- |
1893 |
// The chain (Op#0) and basic block destination (Op#2) are always legal types. |
--- |
| 1894 |
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond, |
1 |
1894 |
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond, |
1 |
| 1895 |
N->getOperand(2)), 0); |
2 |
1895 |
N->getOperand(2)), 0); |
2 |
| 1896 |
} |
--- |
1896 |
} |
--- |
| 1897 |
|
--- |
1897 |
|
--- |
| 1898 |
SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) { |
0 |
1898 |
SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) { |
0 |
| 1899 |
// Since the result type is legal, the operands must promote to it. |
--- |
1899 |
// Since the result type is legal, the operands must promote to it. |
--- |
| 1900 |
EVT OVT = N->getOperand(0).getValueType(); |
0 |
1900 |
EVT OVT = N->getOperand(0).getValueType(); |
0 |
| 1901 |
SDValue Lo = ZExtPromotedInteger(N->getOperand(0)); |
0 |
1901 |
SDValue Lo = ZExtPromotedInteger(N->getOperand(0)); |
0 |
| 1902 |
SDValue Hi = GetPromotedInteger(N->getOperand(1)); |
0 |
1902 |
SDValue Hi = GetPromotedInteger(N->getOperand(1)); |
0 |
| 1903 |
assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?"); |
0 |
1903 |
assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?"); |
0 |
| 1904 |
SDLoc dl(N); |
0 |
1904 |
SDLoc dl(N); |
0 |
| 1905 |
|
--- |
1905 |
|
--- |
| 1906 |
Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi, |
0 |
1906 |
Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi, |
0 |
| 1907 |
DAG.getConstant(OVT.getSizeInBits(), dl, |
0 |
1907 |
DAG.getConstant(OVT.getSizeInBits(), dl, |
0 |
| 1908 |
TLI.getPointerTy(DAG.getDataLayout()))); |
0 |
1908 |
TLI.getPointerTy(DAG.getDataLayout()))); |
0 |
| 1909 |
return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi); |
0 |
1909 |
return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi); |
0 |
| 1910 |
} |
0 |
1910 |
} |
0 |
| 1911 |
|
--- |
1911 |
|
--- |
| 1912 |
SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) { |
0 |
1912 |
SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) { |
0 |
| 1913 |
// The vector type is legal but the element type is not. This implies |
--- |
1913 |
// The vector type is legal but the element type is not. This implies |
--- |
| 1914 |
// that the vector is a power-of-two in length and that the element |
--- |
1914 |
// that the vector is a power-of-two in length and that the element |
--- |
| 1915 |
// type does not have a strange size (eg: it is not i1). |
--- |
1915 |
// type does not have a strange size (eg: it is not i1). |
--- |
| 1916 |
EVT VecVT = N->getValueType(0); |
0 |
1916 |
EVT VecVT = N->getValueType(0); |
0 |
| 1917 |
unsigned NumElts = VecVT.getVectorNumElements(); |
0 |
1917 |
unsigned NumElts = VecVT.getVectorNumElements(); |
0 |
| 1918 |
assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) && |
0 |
1918 |
assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) && |
0 |
| 1919 |
"Legal vector of one illegal element?"); |
--- |
1919 |
"Legal vector of one illegal element?"); |
--- |
| 1920 |
|
--- |
1920 |
|
--- |
| 1921 |
// Promote the inserted value. The type does not need to match the |
--- |
1921 |
// Promote the inserted value. The type does not need to match the |
--- |
| 1922 |
// vector element type. Check that any extra bits introduced will be |
--- |
1922 |
// vector element type. Check that any extra bits introduced will be |
--- |
| 1923 |
// truncated away. |
--- |
1923 |
// truncated away. |
--- |
| 1924 |
assert(N->getOperand(0).getValueSizeInBits() >= |
0 |
1924 |
assert(N->getOperand(0).getValueSizeInBits() >= |
0 |
| 1925 |
N->getValueType(0).getScalarSizeInBits() && |
--- |
1925 |
N->getValueType(0).getScalarSizeInBits() && |
--- |
| 1926 |
"Type of inserted value narrower than vector element type!"); |
--- |
1926 |
"Type of inserted value narrower than vector element type!"); |
--- |
| 1927 |
|
--- |
1927 |
|
--- |
| 1928 |
SmallVector NewOps; |
0 |
1928 |
SmallVector NewOps; |
0 |
| 1929 |
for (unsigned i = 0; i < NumElts; ++i) |
0 |
1929 |
for (unsigned i = 0; i < NumElts; ++i) |
0 |
| 1930 |
NewOps.push_back(GetPromotedInteger(N->getOperand(i))); |
0 |
1930 |
NewOps.push_back(GetPromotedInteger(N->getOperand(i))); |
0 |
| 1931 |
|
--- |
1931 |
|
--- |
| 1932 |
return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); |
0 |
1932 |
return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); |
0 |
| 1933 |
} |
0 |
1933 |
} |
0 |
| 1934 |
|
--- |
1934 |
|
--- |
| 1935 |
SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N, |
0 |
1935 |
SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N, |
0 |
| 1936 |
unsigned OpNo) { |
--- |
1936 |
unsigned OpNo) { |
--- |
| 1937 |
if (OpNo == 1) { |
0 |
1937 |
if (OpNo == 1) { |
0 |
| 1938 |
// Promote the inserted value. This is valid because the type does not |
--- |
1938 |
// Promote the inserted value. This is valid because the type does not |
--- |
| 1939 |
// have to match the vector element type. |
--- |
1939 |
// have to match the vector element type. |
--- |
| 1940 |
|
--- |
1940 |
|
--- |
| 1941 |
// Check that any extra bits introduced will be truncated away. |
--- |
1941 |
// Check that any extra bits introduced will be truncated away. |
--- |
| 1942 |
assert(N->getOperand(1).getValueSizeInBits() >= |
0 |
1942 |
assert(N->getOperand(1).getValueSizeInBits() >= |
0 |
| 1943 |
N->getValueType(0).getScalarSizeInBits() && |
--- |
1943 |
N->getValueType(0).getScalarSizeInBits() && |
--- |
| 1944 |
"Type of inserted value narrower than vector element type!"); |
--- |
1944 |
"Type of inserted value narrower than vector element type!"); |
--- |
| 1945 |
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), |
0 |
1945 |
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), |
0 |
| 1946 |
GetPromotedInteger(N->getOperand(1)), |
0 |
1946 |
GetPromotedInteger(N->getOperand(1)), |
0 |
| 1947 |
N->getOperand(2)), |
0 |
1947 |
N->getOperand(2)), |
0 |
| 1948 |
0); |
0 |
1948 |
0); |
0 |
| 1949 |
} |
--- |
1949 |
} |
--- |
| 1950 |
|
--- |
1950 |
|
--- |
| 1951 |
assert(OpNo == 2 && "Different operand and result vector types?"); |
0 |
1951 |
assert(OpNo == 2 && "Different operand and result vector types?"); |
0 |
| 1952 |
|
--- |
1952 |
|
--- |
| 1953 |
// Promote the index. |
--- |
1953 |
// Promote the index. |
--- |
| 1954 |
SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N), |
0 |
1954 |
SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N), |
0 |
| 1955 |
TLI.getVectorIdxTy(DAG.getDataLayout())); |
0 |
1955 |
TLI.getVectorIdxTy(DAG.getDataLayout())); |
0 |
| 1956 |
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), |
0 |
1956 |
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), |
0 |
| 1957 |
N->getOperand(1), Idx), 0); |
0 |
1957 |
N->getOperand(1), Idx), 0); |
0 |
| 1958 |
} |
--- |
1958 |
} |
--- |
| 1959 |
|
--- |
1959 |
|
--- |
| 1960 |
SDValue DAGTypeLegalizer::PromoteIntOp_ScalarOp(SDNode *N) { |
0 |
1960 |
SDValue DAGTypeLegalizer::PromoteIntOp_ScalarOp(SDNode *N) { |
0 |
| 1961 |
// Integer SPLAT_VECTOR/SCALAR_TO_VECTOR operands are implicitly truncated, |
--- |
1961 |
// Integer SPLAT_VECTOR/SCALAR_TO_VECTOR operands are implicitly truncated, |
--- |
| 1962 |
// so just promote the operand in place. |
--- |
1962 |
// so just promote the operand in place. |
--- |
| 1963 |
return SDValue(DAG.UpdateNodeOperands(N, |
0 |
1963 |
return SDValue(DAG.UpdateNodeOperands(N, |
0 |
| 1964 |
GetPromotedInteger(N->getOperand(0))), 0); |
0 |
1964 |
GetPromotedInteger(N->getOperand(0))), 0); |
0 |
| 1965 |
} |
--- |
1965 |
} |
--- |
| 1966 |
|
--- |
1966 |
|
--- |
| 1967 |
SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) { |
0 |
1967 |
SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) { |
0 |
| 1968 |
assert(OpNo == 0 && "Only know how to promote the condition!"); |
0 |
1968 |
assert(OpNo == 0 && "Only know how to promote the condition!"); |
0 |
| 1969 |
SDValue Cond = N->getOperand(0); |
0 |
1969 |
SDValue Cond = N->getOperand(0); |
0 |
| 1970 |
EVT OpTy = N->getOperand(1).getValueType(); |
0 |
1970 |
EVT OpTy = N->getOperand(1).getValueType(); |
0 |
| 1971 |
|
--- |
1971 |
|
--- |
| 1972 |
if (N->getOpcode() == ISD::VSELECT) |
0 |
1972 |
if (N->getOpcode() == ISD::VSELECT) |
0 |
| 1973 |
if (SDValue Res = WidenVSELECTMask(N)) |
0 |
1973 |
if (SDValue Res = WidenVSELECTMask(N)) |
0 |
| 1974 |
return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), |
0 |
1974 |
return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), |
0 |
| 1975 |
Res, N->getOperand(1), N->getOperand(2)); |
0 |
1975 |
Res, N->getOperand(1), N->getOperand(2)); |
0 |
| 1976 |
|
--- |
1976 |
|
--- |
| 1977 |
// Promote all the way up to the canonical SetCC type. |
--- |
1977 |
// Promote all the way up to the canonical SetCC type. |
--- |
| 1978 |
EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy; |
0 |
1978 |
EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy; |
0 |
| 1979 |
Cond = PromoteTargetBoolean(Cond, OpVT); |
0 |
1979 |
Cond = PromoteTargetBoolean(Cond, OpVT); |
0 |
| 1980 |
|
--- |
1980 |
|
--- |
| 1981 |
return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1), |
0 |
1981 |
return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1), |
0 |
| 1982 |
N->getOperand(2)), 0); |
0 |
1982 |
N->getOperand(2)), 0); |
0 |
| 1983 |
} |
--- |
1983 |
} |
--- |
| 1984 |
|
--- |
1984 |
|
--- |
| 1985 |
SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) { |
0 |
1985 |
SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) { |
0 |
| 1986 |
assert(OpNo == 0 && "Don't know how to promote this operand!"); |
0 |
1986 |
assert(OpNo == 0 && "Don't know how to promote this operand!"); |
0 |
| 1987 |
|
--- |
1987 |
|
--- |
| 1988 |
SDValue LHS = N->getOperand(0); |
0 |
1988 |
SDValue LHS = N->getOperand(0); |
0 |
| 1989 |
SDValue RHS = N->getOperand(1); |
0 |
1989 |
SDValue RHS = N->getOperand(1); |
0 |
| 1990 |
PromoteSetCCOperands(LHS, RHS, cast(N->getOperand(4))->get()); |
0 |
1990 |
PromoteSetCCOperands(LHS, RHS, cast(N->getOperand(4))->get()); |
0 |
| 1991 |
|
--- |
1991 |
|
--- |
| 1992 |
// The CC (#4) and the possible return values (#2 and #3) have legal types. |
--- |
1992 |
// The CC (#4) and the possible return values (#2 and #3) have legal types. |
--- |
| 1993 |
return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2), |
0 |
1993 |
return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2), |
0 |
| 1994 |
N->getOperand(3), N->getOperand(4)), 0); |
0 |
1994 |
N->getOperand(3), N->getOperand(4)), 0); |
0 |
| 1995 |
} |
--- |
1995 |
} |
--- |
| 1996 |
|
--- |
1996 |
|
--- |
| 1997 |
SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) { |
0 |
1997 |
SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) { |
0 |
| 1998 |
assert(OpNo == 0 && "Don't know how to promote this operand!"); |
0 |
1998 |
assert(OpNo == 0 && "Don't know how to promote this operand!"); |
0 |
| 1999 |
|
--- |
1999 |
|
--- |
| 2000 |
SDValue LHS = N->getOperand(0); |
0 |
2000 |
SDValue LHS = N->getOperand(0); |
0 |
| 2001 |
SDValue RHS = N->getOperand(1); |
0 |
2001 |
SDValue RHS = N->getOperand(1); |
0 |
| 2002 |
PromoteSetCCOperands(LHS, RHS, cast(N->getOperand(2))->get()); |
0 |
2002 |
PromoteSetCCOperands(LHS, RHS, cast(N->getOperand(2))->get()); |
0 |
| 2003 |
|
--- |
2003 |
|
--- |
| 2004 |
// The CC (#2) is always legal. |
--- |
2004 |
// The CC (#2) is always legal. |
--- |
| 2005 |
if (N->getOpcode() == ISD::SETCC) |
0 |
2005 |
if (N->getOpcode() == ISD::SETCC) |
0 |
| 2006 |
return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0); |
0 |
2006 |
return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0); |
0 |
| 2007 |
|
--- |
2007 |
|
--- |
| 2008 |
assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode"); |
0 |
2008 |
assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode"); |
0 |
| 2009 |
|
--- |
2009 |
|
--- |
| 2010 |
return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2), |
0 |
2010 |
return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2), |
0 |
| 2011 |
N->getOperand(3), N->getOperand(4)), |
0 |
2011 |
N->getOperand(3), N->getOperand(4)), |
0 |
| 2012 |
0); |
0 |
2012 |
0); |
0 |
| 2013 |
} |
--- |
2013 |
} |
--- |
| 2014 |
|
--- |
2014 |
|
--- |
| 2015 |
SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) { |
0 |
2015 |
SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) { |
0 |
| 2016 |
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), |
0 |
2016 |
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), |
0 |
| 2017 |
ZExtPromotedInteger(N->getOperand(1))), 0); |
0 |
2017 |
ZExtPromotedInteger(N->getOperand(1))), 0); |
0 |
| 2018 |
} |
--- |
2018 |
} |
--- |
| 2019 |
|
--- |
2019 |
|
--- |
| 2020 |
SDValue DAGTypeLegalizer::PromoteIntOp_FunnelShift(SDNode *N) { |
0 |
2020 |
SDValue DAGTypeLegalizer::PromoteIntOp_FunnelShift(SDNode *N) { |
0 |
| 2021 |
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), |
0 |
2021 |
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), |
0 |
| 2022 |
ZExtPromotedInteger(N->getOperand(2))), 0); |
0 |
2022 |
ZExtPromotedInteger(N->getOperand(2))), 0); |
0 |
| 2023 |
} |
--- |
2023 |
} |
--- |
| 2024 |
|
--- |
2024 |
|
--- |
| 2025 |
SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) { |
0 |
2025 |
SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) { |
0 |
| 2026 |
SDValue Op = GetPromotedInteger(N->getOperand(0)); |
0 |
2026 |
SDValue Op = GetPromotedInteger(N->getOperand(0)); |
0 |
| 2027 |
SDLoc dl(N); |
0 |
2027 |
SDLoc dl(N); |
0 |
| 2028 |
Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); |
0 |
2028 |
Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); |
0 |
| 2029 |
return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(), |
0 |
2029 |
return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(), |
0 |
| 2030 |
Op, DAG.getValueType(N->getOperand(0).getValueType())); |
0 |
2030 |
Op, DAG.getValueType(N->getOperand(0).getValueType())); |
0 |
| 2031 |
} |
0 |
2031 |
} |
0 |
| 2032 |
|
--- |
2032 |
|
--- |
| 2033 |
SDValue DAGTypeLegalizer::PromoteIntOp_VP_SIGN_EXTEND(SDNode *N) { |
0 |
2033 |
SDValue DAGTypeLegalizer::PromoteIntOp_VP_SIGN_EXTEND(SDNode *N) { |
0 |
| 2034 |
SDLoc dl(N); |
0 |
2034 |
SDLoc dl(N); |
0 |
| 2035 |
EVT VT = N->getValueType(0); |
0 |
2035 |
EVT VT = N->getValueType(0); |
0 |
| 2036 |
SDValue Op = GetPromotedInteger(N->getOperand(0)); |
0 |
2036 |
SDValue Op = GetPromotedInteger(N->getOperand(0)); |
0 |
| 2037 |
// FIXME: There is no VP_ANY_EXTEND yet. |
--- |
2037 |
// FIXME: There is no VP_ANY_EXTEND yet. |
--- |
| 2038 |
Op = DAG.getNode(ISD::VP_ZERO_EXTEND, dl, VT, Op, N->getOperand(1), |
0 |
2038 |
Op = DAG.getNode(ISD::VP_ZERO_EXTEND, dl, VT, Op, N->getOperand(1), |
0 |
| 2039 |
N->getOperand(2)); |
0 |
2039 |
N->getOperand(2)); |
0 |
| 2040 |
unsigned Diff = |
--- |
2040 |
unsigned Diff = |
--- |
| 2041 |
VT.getScalarSizeInBits() - N->getOperand(0).getScalarValueSizeInBits(); |
0 |
2041 |
VT.getScalarSizeInBits() - N->getOperand(0).getScalarValueSizeInBits(); |
0 |
| 2042 |
SDValue ShAmt = DAG.getShiftAmountConstant(Diff, VT, dl); |
0 |
2042 |
SDValue ShAmt = DAG.getShiftAmountConstant(Diff, VT, dl); |
0 |
| 2043 |
// FIXME: There is no VP_SIGN_EXTEND_INREG so use a pair of shifts. |
--- |
2043 |
// FIXME: There is no VP_SIGN_EXTEND_INREG so use a pair of shifts. |
--- |
| 2044 |
SDValue Shl = DAG.getNode(ISD::VP_SHL, dl, VT, Op, ShAmt, N->getOperand(1), |
0 |
2044 |
SDValue Shl = DAG.getNode(ISD::VP_SHL, dl, VT, Op, ShAmt, N->getOperand(1), |
0 |
| 2045 |
N->getOperand(2)); |
0 |
2045 |
N->getOperand(2)); |
0 |
| 2046 |
return DAG.getNode(ISD::VP_ASHR, dl, VT, Shl, ShAmt, N->getOperand(1), |
0 |
2046 |
return DAG.getNode(ISD::VP_ASHR, dl, VT, Shl, ShAmt, N->getOperand(1), |
0 |
| 2047 |
N->getOperand(2)); |
0 |
2047 |
N->getOperand(2)); |
0 |
| 2048 |
} |
0 |
2048 |
} |
0 |
| 2049 |
|
--- |
2049 |
|
--- |
| 2050 |
SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) { |
0 |
2050 |
SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) { |
0 |
| 2051 |
if (N->getOpcode() == ISD::VP_SINT_TO_FP) |
0 |
2051 |
if (N->getOpcode() == ISD::VP_SINT_TO_FP) |
0 |
| 2052 |
return SDValue(DAG.UpdateNodeOperands(N, |
0 |
2052 |
return SDValue(DAG.UpdateNodeOperands(N, |
0 |
| 2053 |
SExtPromotedInteger(N->getOperand(0)), |
0 |
2053 |
SExtPromotedInteger(N->getOperand(0)), |
0 |
| 2054 |
N->getOperand(1), N->getOperand(2)), |
0 |
2054 |
N->getOperand(1), N->getOperand(2)), |
0 |
| 2055 |
0); |
0 |
2055 |
0); |
0 |
| 2056 |
return SDValue(DAG.UpdateNodeOperands(N, |
0 |
2056 |
return SDValue(DAG.UpdateNodeOperands(N, |
0 |
| 2057 |
SExtPromotedInteger(N->getOperand(0))), 0); |
0 |
2057 |
SExtPromotedInteger(N->getOperand(0))), 0); |
0 |
| 2058 |
} |
--- |
2058 |
} |
--- |
| 2059 |
|
--- |
2059 |
|
--- |
| 2060 |
SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(SDNode *N) { |
0 |
2060 |
SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(SDNode *N) { |
0 |
| 2061 |
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), |
0 |
2061 |
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), |
0 |
| 2062 |
SExtPromotedInteger(N->getOperand(1))), 0); |
0 |
2062 |
SExtPromotedInteger(N->getOperand(1))), 0); |
0 |
| 2063 |
} |
--- |
2063 |
} |
--- |
| 2064 |
|
--- |
2064 |
|
--- |
| 2065 |
SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){ |
0 |
2065 |
SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){ |
0 |
| 2066 |
assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); |
0 |
2066 |
assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); |
0 |
| 2067 |
SDValue Ch = N->getChain(), Ptr = N->getBasePtr(); |
0 |
2067 |
SDValue Ch = N->getChain(), Ptr = N->getBasePtr(); |
0 |
| 2068 |
SDLoc dl(N); |
0 |
2068 |
SDLoc dl(N); |
0 |
| 2069 |
|
--- |
2069 |
|
--- |
| 2070 |
SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value. |
0 |
2070 |
SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value. |
0 |
| 2071 |
|
--- |
2071 |
|
--- |
| 2072 |
// Truncate the value and store the result. |
--- |
2072 |
// Truncate the value and store the result. |
--- |
| 2073 |
return DAG.getTruncStore(Ch, dl, Val, Ptr, |
0 |
2073 |
return DAG.getTruncStore(Ch, dl, Val, Ptr, |
0 |
| 2074 |
N->getMemoryVT(), N->getMemOperand()); |
0 |
2074 |
N->getMemoryVT(), N->getMemOperand()); |
0 |
| 2075 |
} |
0 |
2075 |
} |
0 |
| 2076 |
|
--- |
2076 |
|
--- |
| 2077 |
SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N, |
0 |
2077 |
SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N, |
0 |
| 2078 |
unsigned OpNo) { |
--- |
2078 |
unsigned OpNo) { |
--- |
| 2079 |
SDValue DataOp = N->getValue(); |
0 |
2079 |
SDValue DataOp = N->getValue(); |
0 |
| 2080 |
SDValue Mask = N->getMask(); |
0 |
2080 |
SDValue Mask = N->getMask(); |
0 |
| 2081 |
|
--- |
2081 |
|
--- |
| 2082 |
if (OpNo == 4) { |
0 |
2082 |
if (OpNo == 4) { |
0 |
| 2083 |
// The Mask. Update in place. |
--- |
2083 |
// The Mask. Update in place. |
--- |
| 2084 |
EVT DataVT = DataOp.getValueType(); |
0 |
2084 |
EVT DataVT = DataOp.getValueType(); |
0 |
| 2085 |
Mask = PromoteTargetBoolean(Mask, DataVT); |
0 |
2085 |
Mask = PromoteTargetBoolean(Mask, DataVT); |
0 |
| 2086 |
SmallVector NewOps(N->op_begin(), N->op_end()); |
0 |
2086 |
SmallVector NewOps(N->op_begin(), N->op_end()); |
0 |
| 2087 |
NewOps[4] = Mask; |
0 |
2087 |
NewOps[4] = Mask; |
0 |
| 2088 |
return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); |
0 |
2088 |
return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); |
0 |
| 2089 |
} |
0 |
2089 |
} |
0 |
| 2090 |
|
--- |
2090 |
|
--- |
| 2091 |
assert(OpNo == 1 && "Unexpected operand for promotion"); |
0 |
2091 |
assert(OpNo == 1 && "Unexpected operand for promotion"); |
0 |
| 2092 |
DataOp = GetPromotedInteger(DataOp); |
0 |
2092 |
DataOp = GetPromotedInteger(DataOp); |
0 |
| 2093 |
|
--- |
2093 |
|
--- |
| 2094 |
return DAG.getMaskedStore(N->getChain(), SDLoc(N), DataOp, N->getBasePtr(), |
0 |
2094 |
return DAG.getMaskedStore(N->getChain(), SDLoc(N), DataOp, N->getBasePtr(), |
0 |
| 2095 |
N->getOffset(), Mask, N->getMemoryVT(), |
0 |
2095 |
N->getOffset(), Mask, N->getMemoryVT(), |
0 |
| 2096 |
N->getMemOperand(), N->getAddressingMode(), |
--- |
2096 |
N->getMemOperand(), N->getAddressingMode(), |
--- |
| 2097 |
/*IsTruncating*/ true, N->isCompressingStore()); |
0 |
2097 |
/*IsTruncating*/ true, N->isCompressingStore()); |
0 |
| 2098 |
} |
--- |
2098 |
} |
--- |
| 2099 |
|
--- |
2099 |
|
--- |
| 2100 |
SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N, |
0 |
2100 |
SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N, |
0 |
| 2101 |
unsigned OpNo) { |
--- |
2101 |
unsigned OpNo) { |
--- |
| 2102 |
assert(OpNo == 3 && "Only know how to promote the mask!"); |
0 |
2102 |
assert(OpNo == 3 && "Only know how to promote the mask!"); |
0 |
| 2103 |
EVT DataVT = N->getValueType(0); |
0 |
2103 |
EVT DataVT = N->getValueType(0); |
0 |
| 2104 |
SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); |
0 |
2104 |
SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); |
0 |
| 2105 |
SmallVector NewOps(N->op_begin(), N->op_end()); |
0 |
2105 |
SmallVector NewOps(N->op_begin(), N->op_end()); |
0 |
| 2106 |
NewOps[OpNo] = Mask; |
0 |
2106 |
NewOps[OpNo] = Mask; |
0 |
| 2107 |
SDNode *Res = DAG.UpdateNodeOperands(N, NewOps); |
0 |
2107 |
SDNode *Res = DAG.UpdateNodeOperands(N, NewOps); |
0 |
| 2108 |
if (Res == N) |
0 |
2108 |
if (Res == N) |
0 |
| 2109 |
return SDValue(Res, 0); |
0 |
2109 |
return SDValue(Res, 0); |
0 |
| 2110 |
|
--- |
2110 |
|
--- |
| 2111 |
// Update triggered CSE, do our own replacement since caller can't. |
--- |
2111 |
// Update triggered CSE, do our own replacement since caller can't. |
--- |
| 2112 |
ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0)); |
0 |
2112 |
ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0)); |
0 |
| 2113 |
ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1)); |
0 |
2113 |
ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1)); |
0 |
| 2114 |
return SDValue(); |
0 |
2114 |
return SDValue(); |
0 |
| 2115 |
} |
0 |
2115 |
} |
0 |
| 2116 |
|
--- |
2116 |
|
--- |
| 2117 |
SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N, |
0 |
2117 |
SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N, |
0 |
| 2118 |
unsigned OpNo) { |
--- |
2118 |
unsigned OpNo) { |
--- |
| 2119 |
SmallVector NewOps(N->op_begin(), N->op_end()); |
0 |
2119 |
SmallVector NewOps(N->op_begin(), N->op_end()); |
0 |
| 2120 |
|
--- |
2120 |
|
--- |
| 2121 |
if (OpNo == 2) { |
0 |
2121 |
if (OpNo == 2) { |
0 |
| 2122 |
// The Mask |
--- |
2122 |
// The Mask |
--- |
| 2123 |
EVT DataVT = N->getValueType(0); |
0 |
2123 |
EVT DataVT = N->getValueType(0); |
0 |
| 2124 |
NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); |
0 |
2124 |
NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); |
0 |
| 2125 |
} else if (OpNo == 4) { |
0 |
2125 |
} else if (OpNo == 4) { |
0 |
| 2126 |
// The Index |
--- |
2126 |
// The Index |
--- |
| 2127 |
if (N->isIndexSigned()) |
0 |
2127 |
if (N->isIndexSigned()) |
0 |
| 2128 |
// Need to sign extend the index since the bits will likely be used. |
--- |
2128 |
// Need to sign extend the index since the bits will likely be used. |
--- |
| 2129 |
NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo)); |
0 |
2129 |
NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo)); |
0 |
| 2130 |
else |
--- |
2130 |
else |
--- |
| 2131 |
NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo)); |
0 |
2131 |
NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo)); |
0 |
| 2132 |
} else |
--- |
2132 |
} else |
--- |
| 2133 |
NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo)); |
0 |
2133 |
NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo)); |
0 |
| 2134 |
|
--- |
2134 |
|
--- |
| 2135 |
SDNode *Res = DAG.UpdateNodeOperands(N, NewOps); |
0 |
2135 |
SDNode *Res = DAG.UpdateNodeOperands(N, NewOps); |
0 |
| 2136 |
if (Res == N) |
0 |
2136 |
if (Res == N) |
0 |
| 2137 |
return SDValue(Res, 0); |
0 |
2137 |
return SDValue(Res, 0); |
0 |
| 2138 |
|
--- |
2138 |
|
--- |
| 2139 |
// Update triggered CSE, do our own replacement since caller can't. |
--- |
2139 |
// Update triggered CSE, do our own replacement since caller can't. |
--- |
| 2140 |
ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0)); |
0 |
2140 |
ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0)); |
0 |
| 2141 |
ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1)); |
0 |
2141 |
ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1)); |
0 |
| 2142 |
return SDValue(); |
0 |
2142 |
return SDValue(); |
0 |
| 2143 |
} |
0 |
2143 |
} |
0 |
| 2144 |
|
--- |
2144 |
|
--- |
| 2145 |
SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N, |
0 |
2145 |
SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N, |
0 |
| 2146 |
unsigned OpNo) { |
--- |
2146 |
unsigned OpNo) { |
--- |
| 2147 |
bool TruncateStore = N->isTruncatingStore(); |
0 |
2147 |
bool TruncateStore = N->isTruncatingStore(); |
0 |
| 2148 |
SmallVector NewOps(N->op_begin(), N->op_end()); |
0 |
2148 |
SmallVector NewOps(N->op_begin(), N->op_end()); |
0 |
| 2149 |
|
--- |
2149 |
|
--- |
| 2150 |
if (OpNo == 2) { |
0 |
2150 |
if (OpNo == 2) { |
0 |
| 2151 |
// The Mask |
--- |
2151 |
// The Mask |
--- |
| 2152 |
EVT DataVT = N->getValue().getValueType(); |
0 |
2152 |
EVT DataVT = N->getValue().getValueType(); |
0 |
| 2153 |
NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); |
0 |
2153 |
NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); |
0 |
| 2154 |
} else if (OpNo == 4) { |
0 |
2154 |
} else if (OpNo == 4) { |
0 |
| 2155 |
// The Index |
--- |
2155 |
// The Index |
--- |
| 2156 |
if (N->isIndexSigned()) |
0 |
2156 |
if (N->isIndexSigned()) |
0 |
| 2157 |
// Need to sign extend the index since the bits will likely be used. |
--- |
2157 |
// Need to sign extend the index since the bits will likely be used. |
--- |
| 2158 |
NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo)); |
0 |
2158 |
NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo)); |
0 |
| 2159 |
else |
--- |
2159 |
else |
--- |
| 2160 |
NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo)); |
0 |
2160 |
NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo)); |
0 |
| 2161 |
} else { |
--- |
2161 |
} else { |
--- |
| 2162 |
NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo)); |
0 |
2162 |
NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo)); |
0 |
| 2163 |
TruncateStore = true; |
0 |
2163 |
TruncateStore = true; |
0 |
| 2164 |
} |
--- |
2164 |
} |
--- |
| 2165 |
|
--- |
2165 |
|
--- |
| 2166 |
return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), N->getMemoryVT(), |
0 |
2166 |
return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), N->getMemoryVT(), |
0 |
| 2167 |
SDLoc(N), NewOps, N->getMemOperand(), |
0 |
2167 |
SDLoc(N), NewOps, N->getMemOperand(), |
0 |
| 2168 |
N->getIndexType(), TruncateStore); |
0 |
2168 |
N->getIndexType(), TruncateStore); |
0 |
| 2169 |
} |
0 |
2169 |
} |
0 |
| 2170 |
|
--- |
2170 |
|
--- |
| 2171 |
SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) { |
0 |
2171 |
SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) { |
0 |
| 2172 |
SDValue Op = GetPromotedInteger(N->getOperand(0)); |
0 |
2172 |
SDValue Op = GetPromotedInteger(N->getOperand(0)); |
0 |
| 2173 |
if (N->getOpcode() == ISD::VP_TRUNCATE) |
0 |
2173 |
if (N->getOpcode() == ISD::VP_TRUNCATE) |
0 |
| 2174 |
return DAG.getNode(ISD::VP_TRUNCATE, SDLoc(N), N->getValueType(0), Op, |
0 |
2174 |
return DAG.getNode(ISD::VP_TRUNCATE, SDLoc(N), N->getValueType(0), Op, |
0 |
| 2175 |
N->getOperand(1), N->getOperand(2)); |
0 |
2175 |
N->getOperand(1), N->getOperand(2)); |
0 |
| 2176 |
return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op); |
0 |
2176 |
return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op); |
0 |
| 2177 |
} |
--- |
2177 |
} |
--- |
| 2178 |
|
--- |
2178 |
|
--- |
| 2179 |
SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) { |
0 |
2179 |
SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) { |
0 |
| 2180 |
if (N->getOpcode() == ISD::VP_UINT_TO_FP) |
0 |
2180 |
if (N->getOpcode() == ISD::VP_UINT_TO_FP) |
0 |
| 2181 |
return SDValue(DAG.UpdateNodeOperands(N, |
0 |
2181 |
return SDValue(DAG.UpdateNodeOperands(N, |
0 |
| 2182 |
ZExtPromotedInteger(N->getOperand(0)), |
0 |
2182 |
ZExtPromotedInteger(N->getOperand(0)), |
0 |
| 2183 |
N->getOperand(1), N->getOperand(2)), |
0 |
2183 |
N->getOperand(1), N->getOperand(2)), |
0 |
| 2184 |
0); |
0 |
2184 |
0); |
0 |
| 2185 |
return SDValue(DAG.UpdateNodeOperands(N, |
0 |
2185 |
return SDValue(DAG.UpdateNodeOperands(N, |
0 |
| 2186 |
ZExtPromotedInteger(N->getOperand(0))), 0); |
0 |
2186 |
ZExtPromotedInteger(N->getOperand(0))), 0); |
0 |
| 2187 |
} |
--- |
2187 |
} |
--- |
| 2188 |
|
--- |
2188 |
|
--- |
| 2189 |
SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(SDNode *N) { |
0 |
2189 |
SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(SDNode *N) { |
0 |
| 2190 |
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), |
0 |
2190 |
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), |
0 |
| 2191 |
ZExtPromotedInteger(N->getOperand(1))), 0); |
0 |
2191 |
ZExtPromotedInteger(N->getOperand(1))), 0); |
0 |
| 2192 |
} |
--- |
2192 |
} |
--- |
| 2193 |
|
--- |
2193 |
|
--- |
| 2194 |
SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) { |
1 |
2194 |
SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) { |
1 |
| 2195 |
SDLoc dl(N); |
1 |
2195 |
SDLoc dl(N); |
1 |
| 2196 |
SDValue Op = GetPromotedInteger(N->getOperand(0)); |
1 |
2196 |
SDValue Op = GetPromotedInteger(N->getOperand(0)); |
1 |
| 2197 |
Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); |
1 |
2197 |
Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); |
1 |
| 2198 |
return DAG.getZeroExtendInReg(Op, dl, N->getOperand(0).getValueType()); |
1 |
2198 |
return DAG.getZeroExtendInReg(Op, dl, N->getOperand(0).getValueType()); |
1 |
| 2199 |
} |
1 |
2199 |
} |
1 |
| 2200 |
|
--- |
2200 |
|
--- |
| 2201 |
SDValue DAGTypeLegalizer::PromoteIntOp_VP_ZERO_EXTEND(SDNode *N) { |
0 |
2201 |
SDValue DAGTypeLegalizer::PromoteIntOp_VP_ZERO_EXTEND(SDNode *N) { |
0 |
| 2202 |
SDLoc dl(N); |
0 |
2202 |
SDLoc dl(N); |
0 |
| 2203 |
EVT VT = N->getValueType(0); |
0 |
2203 |
EVT VT = N->getValueType(0); |
0 |
| 2204 |
SDValue Op = GetPromotedInteger(N->getOperand(0)); |
0 |
2204 |
SDValue Op = GetPromotedInteger(N->getOperand(0)); |
0 |
| 2205 |
// FIXME: There is no VP_ANY_EXTEND yet. |
--- |
2205 |
// FIXME: There is no VP_ANY_EXTEND yet. |
--- |
| 2206 |
Op = DAG.getNode(ISD::VP_ZERO_EXTEND, dl, VT, Op, N->getOperand(1), |
0 |
2206 |
Op = DAG.getNode(ISD::VP_ZERO_EXTEND, dl, VT, Op, N->getOperand(1), |
0 |
| 2207 |
N->getOperand(2)); |
0 |
2207 |
N->getOperand(2)); |
0 |
| 2208 |
APInt Imm = APInt::getLowBitsSet(VT.getScalarSizeInBits(), |
0 |
2208 |
APInt Imm = APInt::getLowBitsSet(VT.getScalarSizeInBits(), |
0 |
| 2209 |
N->getOperand(0).getScalarValueSizeInBits()); |
0 |
2209 |
N->getOperand(0).getScalarValueSizeInBits()); |
0 |
| 2210 |
return DAG.getNode(ISD::VP_AND, dl, VT, Op, DAG.getConstant(Imm, dl, VT), |
0 |
2210 |
return DAG.getNode(ISD::VP_AND, dl, VT, Op, DAG.getConstant(Imm, dl, VT), |
0 |
| 2211 |
N->getOperand(1), N->getOperand(2)); |
0 |
2211 |
N->getOperand(1), N->getOperand(2)); |
0 |
| 2212 |
} |
0 |
2212 |
} |
0 |
| 2213 |
|
--- |
2213 |
|
--- |
| 2214 |
SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBO_CARRY(SDNode *N, unsigned OpNo) { |
0 |
2214 |
SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBO_CARRY(SDNode *N, unsigned OpNo) { |
0 |
| 2215 |
assert(OpNo == 2 && "Don't know how to promote this operand!"); |
0 |
2215 |
assert(OpNo == 2 && "Don't know how to promote this operand!"); |
0 |
| 2216 |
|
--- |
2216 |
|
--- |
| 2217 |
SDValue LHS = N->getOperand(0); |
0 |
2217 |
SDValue LHS = N->getOperand(0); |
0 |
| 2218 |
SDValue RHS = N->getOperand(1); |
0 |
2218 |
SDValue RHS = N->getOperand(1); |
0 |
| 2219 |
SDValue Carry = N->getOperand(2); |
0 |
2219 |
SDValue Carry = N->getOperand(2); |
0 |
| 2220 |
SDLoc DL(N); |
0 |
2220 |
SDLoc DL(N); |
0 |
| 2221 |
|
--- |
2221 |
|
--- |
| 2222 |
Carry = PromoteTargetBoolean(Carry, LHS.getValueType()); |
0 |
2222 |
Carry = PromoteTargetBoolean(Carry, LHS.getValueType()); |
0 |
| 2223 |
|
--- |
2223 |
|
--- |
| 2224 |
return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0); |
0 |
2224 |
return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0); |
0 |
| 2225 |
} |
0 |
2225 |
} |
0 |
| 2226 |
|
--- |
2226 |
|
--- |
| 2227 |
SDValue DAGTypeLegalizer::PromoteIntOp_FIX(SDNode *N) { |
0 |
2227 |
SDValue DAGTypeLegalizer::PromoteIntOp_FIX(SDNode *N) { |
0 |
| 2228 |
SDValue Op2 = ZExtPromotedInteger(N->getOperand(2)); |
0 |
2228 |
SDValue Op2 = ZExtPromotedInteger(N->getOperand(2)); |
0 |
| 2229 |
return SDValue( |
0 |
2229 |
return SDValue( |
0 |
| 2230 |
DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0); |
0 |
2230 |
DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0); |
0 |
| 2231 |
} |
--- |
2231 |
} |
--- |
| 2232 |
|
--- |
2232 |
|
--- |
| 2233 |
SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) { |
0 |
2233 |
SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) { |
0 |
| 2234 |
// Promote the RETURNADDR/FRAMEADDR argument to a supported integer width. |
--- |
2234 |
// Promote the RETURNADDR/FRAMEADDR argument to a supported integer width. |
--- |
| 2235 |
SDValue Op = ZExtPromotedInteger(N->getOperand(0)); |
0 |
2235 |
SDValue Op = ZExtPromotedInteger(N->getOperand(0)); |
0 |
| 2236 |
return SDValue(DAG.UpdateNodeOperands(N, Op), 0); |
0 |
2236 |
return SDValue(DAG.UpdateNodeOperands(N, Op), 0); |
0 |
| 2237 |
} |
--- |
2237 |
} |
--- |
| 2238 |
|
--- |
2238 |
|
--- |
| 2239 |
SDValue DAGTypeLegalizer::PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo) { |
0 |
2239 |
SDValue DAGTypeLegalizer::PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo) { |
0 |
| 2240 |
assert(OpNo > 1 && "Don't know how to promote this operand!"); |
0 |
2240 |
assert(OpNo > 1 && "Don't know how to promote this operand!"); |
0 |
| 2241 |
// Promote the rw, locality, and cache type arguments to a supported integer |
--- |
2241 |
// Promote the rw, locality, and cache type arguments to a supported integer |
--- |
| 2242 |
// width. |
--- |
2242 |
// width. |
--- |
| 2243 |
SDValue Op2 = ZExtPromotedInteger(N->getOperand(2)); |
0 |
2243 |
SDValue Op2 = ZExtPromotedInteger(N->getOperand(2)); |
0 |
| 2244 |
SDValue Op3 = ZExtPromotedInteger(N->getOperand(3)); |
0 |
2244 |
SDValue Op3 = ZExtPromotedInteger(N->getOperand(3)); |
0 |
| 2245 |
SDValue Op4 = ZExtPromotedInteger(N->getOperand(4)); |
0 |
2245 |
SDValue Op4 = ZExtPromotedInteger(N->getOperand(4)); |
0 |
| 2246 |
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), |
0 |
2246 |
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), |
0 |
| 2247 |
Op2, Op3, Op4), |
--- |
2247 |
Op2, Op3, Op4), |
--- |
| 2248 |
0); |
0 |
2248 |
0); |
0 |
| 2249 |
} |
--- |
2249 |
} |
--- |
| 2250 |
|
--- |
2250 |
|
--- |
| 2251 |
SDValue DAGTypeLegalizer::PromoteIntOp_ExpOp(SDNode *N) { |
0 |
2251 |
SDValue DAGTypeLegalizer::PromoteIntOp_ExpOp(SDNode *N) { |
0 |
| 2252 |
bool IsStrict = N->isStrictFPOpcode(); |
0 |
2252 |
bool IsStrict = N->isStrictFPOpcode(); |
0 |
| 2253 |
SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); |
0 |
2253 |
SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); |
0 |
| 2254 |
|
--- |
2254 |
|
--- |
| 2255 |
bool IsPowI = |
--- |
2255 |
bool IsPowI = |
--- |
| 2256 |
N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI; |
0 |
2256 |
N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI; |
0 |
| 2257 |
|
--- |
2257 |
|
--- |
| 2258 |
// The integer operand is the last operand in FPOWI (or FLDEXP) (so the result |
--- |
2258 |
// The integer operand is the last operand in FPOWI (or FLDEXP) (so the result |
--- |
| 2259 |
// and floating point operand is already type legalized). |
--- |
2259 |
// and floating point operand is already type legalized). |
--- |
| 2260 |
RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(N->getValueType(0)) |
0 |
2260 |
RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(N->getValueType(0)) |
0 |
| 2261 |
: RTLIB::getLDEXP(N->getValueType(0)); |
0 |
2261 |
: RTLIB::getLDEXP(N->getValueType(0)); |
0 |
| 2262 |
|
--- |
2262 |
|
--- |
| 2263 |
if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) { |
0 |
2263 |
if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) { |
0 |
| 2264 |
SDValue Op = SExtPromotedInteger(N->getOperand(1)); |
0 |
2264 |
SDValue Op = SExtPromotedInteger(N->getOperand(1)); |
0 |
| 2265 |
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0); |
0 |
2265 |
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0); |
0 |
| 2266 |
} |
--- |
2266 |
} |
--- |
| 2267 |
|
--- |
2267 |
|
--- |
| 2268 |
// We can't just promote the exponent type in FPOWI, since we want to lower |
--- |
2268 |
// We can't just promote the exponent type in FPOWI, since we want to lower |
--- |
| 2269 |
// the node to a libcall and we if we promote to a type larger than |
--- |
2269 |
// the node to a libcall and we if we promote to a type larger than |
--- |
| 2270 |
// sizeof(int) the libcall might not be according to the targets ABI. Instead |
--- |
2270 |
// sizeof(int) the libcall might not be according to the targets ABI. Instead |
--- |
| 2271 |
// we rewrite to a libcall here directly, letting makeLibCall handle promotion |
--- |
2271 |
// we rewrite to a libcall here directly, letting makeLibCall handle promotion |
--- |
| 2272 |
// if the target accepts it according to shouldSignExtendTypeInLibCall. |
--- |
2272 |
// if the target accepts it according to shouldSignExtendTypeInLibCall. |
--- |
| 2273 |
|
--- |
2273 |
|
--- |
| 2274 |
unsigned OpOffset = IsStrict ? 1 : 0; |
0 |
2274 |
unsigned OpOffset = IsStrict ? 1 : 0; |
0 |
| 2275 |
// The exponent should fit in a sizeof(int) type for the libcall to be valid. |
--- |
2275 |
// The exponent should fit in a sizeof(int) type for the libcall to be valid. |
--- |
| 2276 |
assert(DAG.getLibInfo().getIntSize() == |
0 |
2276 |
assert(DAG.getLibInfo().getIntSize() == |
0 |
| 2277 |
N->getOperand(1 + OpOffset).getValueType().getSizeInBits() && |
--- |
2277 |
N->getOperand(1 + OpOffset).getValueType().getSizeInBits() && |
--- |
| 2278 |
"POWI exponent should match with sizeof(int) when doing the libcall."); |
--- |
2278 |
"POWI exponent should match with sizeof(int) when doing the libcall."); |
--- |
| 2279 |
TargetLowering::MakeLibCallOptions CallOptions; |
0 |
2279 |
TargetLowering::MakeLibCallOptions CallOptions; |
0 |
| 2280 |
CallOptions.setSExt(true); |
0 |
2280 |
CallOptions.setSExt(true); |
0 |
| 2281 |
SDValue Ops[2] = {N->getOperand(0 + OpOffset), N->getOperand(1 + OpOffset)}; |
0 |
2281 |
SDValue Ops[2] = {N->getOperand(0 + OpOffset), N->getOperand(1 + OpOffset)}; |
0 |
| 2282 |
std::pair Tmp = TLI.makeLibCall( |
0 |
2282 |
std::pair Tmp = TLI.makeLibCall( |
0 |
| 2283 |
DAG, LC, N->getValueType(0), Ops, CallOptions, SDLoc(N), Chain); |
0 |
2283 |
DAG, LC, N->getValueType(0), Ops, CallOptions, SDLoc(N), Chain); |
0 |
| 2284 |
ReplaceValueWith(SDValue(N, 0), Tmp.first); |
0 |
2284 |
ReplaceValueWith(SDValue(N, 0), Tmp.first); |
0 |
| 2285 |
if (IsStrict) |
0 |
2285 |
if (IsStrict) |
0 |
| 2286 |
ReplaceValueWith(SDValue(N, 1), Tmp.second); |
0 |
2286 |
ReplaceValueWith(SDValue(N, 1), Tmp.second); |
0 |
| 2287 |
return SDValue(); |
0 |
2287 |
return SDValue(); |
0 |
| 2288 |
} |
--- |
2288 |
} |
--- |
| 2289 |
|
--- |
2289 |
|
--- |
| 2290 |
static unsigned getExtendForIntVecReduction(SDNode *N) { |
0 |
2290 |
static unsigned getExtendForIntVecReduction(SDNode *N) { |
0 |
| 2291 |
switch (N->getOpcode()) { |
0 |
2291 |
switch (N->getOpcode()) { |
0 |
| 2292 |
default: |
0 |
2292 |
default: |
0 |
| 2293 |
llvm_unreachable("Expected integer vector reduction"); |
0 |
2293 |
llvm_unreachable("Expected integer vector reduction"); |
0 |
| 2294 |
case ISD::VECREDUCE_ADD: |
0 |
2294 |
case ISD::VECREDUCE_ADD: |
0 |
| 2295 |
case ISD::VECREDUCE_MUL: |
--- |
2295 |
case ISD::VECREDUCE_MUL: |
--- |
| 2296 |
case ISD::VECREDUCE_AND: |
--- |
2296 |
case ISD::VECREDUCE_AND: |
--- |
| 2297 |
case ISD::VECREDUCE_OR: |
--- |
2297 |
case ISD::VECREDUCE_OR: |
--- |
| 2298 |
case ISD::VECREDUCE_XOR: |
--- |
2298 |
case ISD::VECREDUCE_XOR: |
--- |
| 2299 |
case ISD::VP_REDUCE_ADD: |
--- |
2299 |
case ISD::VP_REDUCE_ADD: |
--- |
| 2300 |
case ISD::VP_REDUCE_MUL: |
--- |
2300 |
case ISD::VP_REDUCE_MUL: |
--- |
| 2301 |
case ISD::VP_REDUCE_AND: |
--- |
2301 |
case ISD::VP_REDUCE_AND: |
--- |
| 2302 |
case ISD::VP_REDUCE_OR: |
--- |
2302 |
case ISD::VP_REDUCE_OR: |
--- |
| 2303 |
case ISD::VP_REDUCE_XOR: |
--- |
2303 |
case ISD::VP_REDUCE_XOR: |
--- |
| 2304 |
return ISD::ANY_EXTEND; |
0 |
2304 |
return ISD::ANY_EXTEND; |
0 |
| 2305 |
case ISD::VECREDUCE_SMAX: |
0 |
2305 |
case ISD::VECREDUCE_SMAX: |
0 |
| 2306 |
case ISD::VECREDUCE_SMIN: |
--- |
2306 |
case ISD::VECREDUCE_SMIN: |
--- |
| 2307 |
case ISD::VP_REDUCE_SMAX: |
--- |
2307 |
case ISD::VP_REDUCE_SMAX: |
--- |
| 2308 |
case ISD::VP_REDUCE_SMIN: |
--- |
2308 |
case ISD::VP_REDUCE_SMIN: |
--- |
| 2309 |
return ISD::SIGN_EXTEND; |
0 |
2309 |
return ISD::SIGN_EXTEND; |
0 |
| 2310 |
case ISD::VECREDUCE_UMAX: |
0 |
2310 |
case ISD::VECREDUCE_UMAX: |
0 |
| 2311 |
case ISD::VECREDUCE_UMIN: |
--- |
2311 |
case ISD::VECREDUCE_UMIN: |
--- |
| 2312 |
case ISD::VP_REDUCE_UMAX: |
--- |
2312 |
case ISD::VP_REDUCE_UMAX: |
--- |
| 2313 |
case ISD::VP_REDUCE_UMIN: |
--- |
2313 |
case ISD::VP_REDUCE_UMIN: |
--- |
| 2314 |
return ISD::ZERO_EXTEND; |
0 |
2314 |
return ISD::ZERO_EXTEND; |
0 |
| 2315 |
} |
--- |
2315 |
} |
--- |
| 2316 |
} |
--- |
2316 |
} |
--- |
| 2317 |
|
--- |
2317 |
|
--- |
| 2318 |
SDValue DAGTypeLegalizer::PromoteIntOpVectorReduction(SDNode *N, SDValue V) { |
0 |
2318 |
SDValue DAGTypeLegalizer::PromoteIntOpVectorReduction(SDNode *N, SDValue V) { |
0 |
| 2319 |
switch (getExtendForIntVecReduction(N)) { |
0 |
2319 |
switch (getExtendForIntVecReduction(N)) { |
0 |
| 2320 |
default: |
0 |
2320 |
default: |
0 |
| 2321 |
llvm_unreachable("Impossible extension kind for integer reduction"); |
0 |
2321 |
llvm_unreachable("Impossible extension kind for integer reduction"); |
0 |
| 2322 |
case ISD::ANY_EXTEND: |
0 |
2322 |
case ISD::ANY_EXTEND: |
0 |
| 2323 |
return GetPromotedInteger(V); |
0 |
2323 |
return GetPromotedInteger(V); |
0 |
| 2324 |
case ISD::SIGN_EXTEND: |
0 |
2324 |
case ISD::SIGN_EXTEND: |
0 |
| 2325 |
return SExtPromotedInteger(V); |
0 |
2325 |
return SExtPromotedInteger(V); |
0 |
| 2326 |
case ISD::ZERO_EXTEND: |
0 |
2326 |
case ISD::ZERO_EXTEND: |
0 |
| 2327 |
return ZExtPromotedInteger(V); |
0 |
2327 |
return ZExtPromotedInteger(V); |
0 |
| 2328 |
} |
--- |
2328 |
} |
--- |
| 2329 |
} |
--- |
2329 |
} |
--- |
| 2330 |
|
--- |
2330 |
|
--- |
| 2331 |
SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) { |
0 |
2331 |
SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) { |
0 |
| 2332 |
SDLoc dl(N); |
0 |
2332 |
SDLoc dl(N); |
0 |
| 2333 |
SDValue Op = PromoteIntOpVectorReduction(N, N->getOperand(0)); |
0 |
2333 |
SDValue Op = PromoteIntOpVectorReduction(N, N->getOperand(0)); |
0 |
| 2334 |
|
--- |
2334 |
|
--- |
| 2335 |
EVT OrigEltVT = N->getOperand(0).getValueType().getVectorElementType(); |
0 |
2335 |
EVT OrigEltVT = N->getOperand(0).getValueType().getVectorElementType(); |
0 |
| 2336 |
EVT InVT = Op.getValueType(); |
0 |
2336 |
EVT InVT = Op.getValueType(); |
0 |
| 2337 |
EVT EltVT = InVT.getVectorElementType(); |
0 |
2337 |
EVT EltVT = InVT.getVectorElementType(); |
0 |
| 2338 |
EVT ResVT = N->getValueType(0); |
0 |
2338 |
EVT ResVT = N->getValueType(0); |
0 |
| 2339 |
unsigned Opcode = N->getOpcode(); |
0 |
2339 |
unsigned Opcode = N->getOpcode(); |
0 |
| 2340 |
|
--- |
2340 |
|
--- |
| 2341 |
// An i1 vecreduce_xor is equivalent to vecreduce_add, use that instead if |
--- |
2341 |
// An i1 vecreduce_xor is equivalent to vecreduce_add, use that instead if |
--- |
| 2342 |
// vecreduce_xor is not legal |
--- |
2342 |
// vecreduce_xor is not legal |
--- |
| 2343 |
if (Opcode == ISD::VECREDUCE_XOR && OrigEltVT == MVT::i1 && |
0 |
2343 |
if (Opcode == ISD::VECREDUCE_XOR && OrigEltVT == MVT::i1 && |
0 |
| 2344 |
!TLI.isOperationLegalOrCustom(ISD::VECREDUCE_XOR, InVT) && |
0 |
2344 |
!TLI.isOperationLegalOrCustom(ISD::VECREDUCE_XOR, InVT) && |
0 |
| 2345 |
TLI.isOperationLegalOrCustom(ISD::VECREDUCE_ADD, InVT)) |
0 |
2345 |
TLI.isOperationLegalOrCustom(ISD::VECREDUCE_ADD, InVT)) |
0 |
| 2346 |
Opcode = ISD::VECREDUCE_ADD; |
0 |
2346 |
Opcode = ISD::VECREDUCE_ADD; |
0 |
| 2347 |
|
--- |
2347 |
|
--- |
| 2348 |
// An i1 vecreduce_or is equivalent to vecreduce_umax, use that instead if |
--- |
2348 |
// An i1 vecreduce_or is equivalent to vecreduce_umax, use that instead if |
--- |
| 2349 |
// vecreduce_or is not legal |
--- |
2349 |
// vecreduce_or is not legal |
--- |
| 2350 |
else if (Opcode == ISD::VECREDUCE_OR && OrigEltVT == MVT::i1 && |
0 |
2350 |
else if (Opcode == ISD::VECREDUCE_OR && OrigEltVT == MVT::i1 && |
0 |
| 2351 |
!TLI.isOperationLegalOrCustom(ISD::VECREDUCE_OR, InVT) && |
0 |
2351 |
!TLI.isOperationLegalOrCustom(ISD::VECREDUCE_OR, InVT) && |
0 |
| 2352 |
TLI.isOperationLegalOrCustom(ISD::VECREDUCE_UMAX, InVT)) { |
0 |
2352 |
TLI.isOperationLegalOrCustom(ISD::VECREDUCE_UMAX, InVT)) { |
0 |
| 2353 |
Opcode = ISD::VECREDUCE_UMAX; |
0 |
2353 |
Opcode = ISD::VECREDUCE_UMAX; |
0 |
| 2354 |
// Can't use promoteTargetBoolean here because we still need |
--- |
2354 |
// Can't use promoteTargetBoolean here because we still need |
--- |
| 2355 |
// to either sign_ext or zero_ext in the undefined case. |
--- |
2355 |
// to either sign_ext or zero_ext in the undefined case. |
--- |
| 2356 |
switch (TLI.getBooleanContents(InVT)) { |
0 |
2356 |
switch (TLI.getBooleanContents(InVT)) { |
0 |
| 2357 |
case TargetLoweringBase::UndefinedBooleanContent: |
0 |
2357 |
case TargetLoweringBase::UndefinedBooleanContent: |
0 |
| 2358 |
case TargetLoweringBase::ZeroOrOneBooleanContent: |
--- |
2358 |
case TargetLoweringBase::ZeroOrOneBooleanContent: |
--- |
| 2359 |
Op = ZExtPromotedInteger(N->getOperand(0)); |
0 |
2359 |
Op = ZExtPromotedInteger(N->getOperand(0)); |
0 |
| 2360 |
break; |
0 |
2360 |
break; |
0 |
| 2361 |
case TargetLoweringBase::ZeroOrNegativeOneBooleanContent: |
0 |
2361 |
case TargetLoweringBase::ZeroOrNegativeOneBooleanContent: |
0 |
| 2362 |
Op = SExtPromotedInteger(N->getOperand(0)); |
0 |
2362 |
Op = SExtPromotedInteger(N->getOperand(0)); |
0 |
| 2363 |
break; |
0 |
2363 |
break; |
0 |
| 2364 |
} |
--- |
2364 |
} |
--- |
| 2365 |
} |
--- |
2365 |
} |
--- |
| 2366 |
|
--- |
2366 |
|
--- |
| 2367 |
// An i1 vecreduce_and is equivalent to vecreduce_umin, use that instead if |
--- |
2367 |
// An i1 vecreduce_and is equivalent to vecreduce_umin, use that instead if |
--- |
| 2368 |
// vecreduce_and is not legal |
--- |
2368 |
// vecreduce_and is not legal |
--- |
| 2369 |
else if (Opcode == ISD::VECREDUCE_AND && OrigEltVT == MVT::i1 && |
0 |
2369 |
else if (Opcode == ISD::VECREDUCE_AND && OrigEltVT == MVT::i1 && |
0 |
| 2370 |
!TLI.isOperationLegalOrCustom(ISD::VECREDUCE_AND, InVT) && |
0 |
2370 |
!TLI.isOperationLegalOrCustom(ISD::VECREDUCE_AND, InVT) && |
0 |
| 2371 |
TLI.isOperationLegalOrCustom(ISD::VECREDUCE_UMIN, InVT)) { |
0 |
2371 |
TLI.isOperationLegalOrCustom(ISD::VECREDUCE_UMIN, InVT)) { |
0 |
| 2372 |
Opcode = ISD::VECREDUCE_UMIN; |
0 |
2372 |
Opcode = ISD::VECREDUCE_UMIN; |
0 |
| 2373 |
// Can't use promoteTargetBoolean here because we still need |
--- |
2373 |
// Can't use promoteTargetBoolean here because we still need |
--- |
| 2374 |
// to either sign_ext or zero_ext in the undefined case. |
--- |
2374 |
// to either sign_ext or zero_ext in the undefined case. |
--- |
| 2375 |
switch (TLI.getBooleanContents(InVT)) { |
0 |
2375 |
switch (TLI.getBooleanContents(InVT)) { |
0 |
| 2376 |
case TargetLoweringBase::UndefinedBooleanContent: |
0 |
2376 |
case TargetLoweringBase::UndefinedBooleanContent: |
0 |
| 2377 |
case TargetLoweringBase::ZeroOrOneBooleanContent: |
--- |
2377 |
case TargetLoweringBase::ZeroOrOneBooleanContent: |
--- |
| 2378 |
Op = ZExtPromotedInteger(N->getOperand(0)); |
0 |
2378 |
Op = ZExtPromotedInteger(N->getOperand(0)); |
0 |
| 2379 |
break; |
0 |
2379 |
break; |
0 |
| 2380 |
case TargetLoweringBase::ZeroOrNegativeOneBooleanContent: |
0 |
2380 |
case TargetLoweringBase::ZeroOrNegativeOneBooleanContent: |
0 |
| 2381 |
Op = SExtPromotedInteger(N->getOperand(0)); |
0 |
2381 |
Op = SExtPromotedInteger(N->getOperand(0)); |
0 |
| 2382 |
break; |
0 |
2382 |
break; |
0 |
| 2383 |
} |
--- |
2383 |
} |
--- |
| 2384 |
} |
--- |
2384 |
} |
--- |
| 2385 |
|
--- |
2385 |
|
--- |
| 2386 |
if (ResVT.bitsGE(EltVT)) |
0 |
2386 |
if (ResVT.bitsGE(EltVT)) |
0 |
| 2387 |
return DAG.getNode(Opcode, SDLoc(N), ResVT, Op); |
0 |
2387 |
return DAG.getNode(Opcode, SDLoc(N), ResVT, Op); |
0 |
| 2388 |
|
--- |
2388 |
|
--- |
| 2389 |
// Result size must be >= element size. If this is not the case after |
--- |
2389 |
// Result size must be >= element size. If this is not the case after |
--- |
| 2390 |
// promotion, also promote the result type and then truncate. |
--- |
2390 |
// promotion, also promote the result type and then truncate. |
--- |
| 2391 |
SDValue Reduce = DAG.getNode(Opcode, dl, EltVT, Op); |
0 |
2391 |
SDValue Reduce = DAG.getNode(Opcode, dl, EltVT, Op); |
0 |
| 2392 |
return DAG.getNode(ISD::TRUNCATE, dl, ResVT, Reduce); |
0 |
2392 |
return DAG.getNode(ISD::TRUNCATE, dl, ResVT, Reduce); |
0 |
| 2393 |
} |
0 |
2393 |
} |
0 |
| 2394 |
|
--- |
2394 |
|
--- |
| 2395 |
SDValue DAGTypeLegalizer::PromoteIntOp_VP_REDUCE(SDNode *N, unsigned OpNo) { |
0 |
2395 |
SDValue DAGTypeLegalizer::PromoteIntOp_VP_REDUCE(SDNode *N, unsigned OpNo) { |
0 |
| 2396 |
SDLoc DL(N); |
0 |
2396 |
SDLoc DL(N); |
0 |
| 2397 |
SDValue Op = N->getOperand(OpNo); |
0 |
2397 |
SDValue Op = N->getOperand(OpNo); |
0 |
| 2398 |
SmallVector NewOps(N->op_begin(), N->op_end()); |
0 |
2398 |
SmallVector NewOps(N->op_begin(), N->op_end()); |
0 |
| 2399 |
|
--- |
2399 |
|
--- |
| 2400 |
if (OpNo == 2) { // Mask |
0 |
2400 |
if (OpNo == 2) { // Mask |
0 |
| 2401 |
// Update in place. |
--- |
2401 |
// Update in place. |
--- |
| 2402 |
NewOps[2] = PromoteTargetBoolean(Op, N->getOperand(1).getValueType()); |
0 |
2402 |
NewOps[2] = PromoteTargetBoolean(Op, N->getOperand(1).getValueType()); |
0 |
| 2403 |
return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); |
0 |
2403 |
return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); |
0 |
| 2404 |
} |
--- |
2404 |
} |
--- |
| 2405 |
|
--- |
2405 |
|
--- |
| 2406 |
assert(OpNo == 1 && "Unexpected operand for promotion"); |
0 |
2406 |
assert(OpNo == 1 && "Unexpected operand for promotion"); |
0 |
| 2407 |
|
--- |
2407 |
|
--- |
| 2408 |
Op = PromoteIntOpVectorReduction(N, Op); |
0 |
2408 |
Op = PromoteIntOpVectorReduction(N, Op); |
0 |
| 2409 |
|
--- |
2409 |
|
--- |
| 2410 |
NewOps[OpNo] = Op; |
0 |
2410 |
NewOps[OpNo] = Op; |
0 |
| 2411 |
|
--- |
2411 |
|
--- |
| 2412 |
EVT VT = N->getValueType(0); |
0 |
2412 |
EVT VT = N->getValueType(0); |
0 |
| 2413 |
EVT EltVT = Op.getValueType().getScalarType(); |
0 |
2413 |
EVT EltVT = Op.getValueType().getScalarType(); |
0 |
| 2414 |
|
--- |
2414 |
|
--- |
| 2415 |
if (VT.bitsGE(EltVT)) |
0 |
2415 |
if (VT.bitsGE(EltVT)) |
0 |
| 2416 |
return DAG.getNode(N->getOpcode(), SDLoc(N), VT, NewOps); |
0 |
2416 |
return DAG.getNode(N->getOpcode(), SDLoc(N), VT, NewOps); |
0 |
| 2417 |
|
--- |
2417 |
|
--- |
| 2418 |
// Result size must be >= element/start-value size. If this is not the case |
--- |
2418 |
// Result size must be >= element/start-value size. If this is not the case |
--- |
| 2419 |
// after promotion, also promote both the start value and result type and |
--- |
2419 |
// after promotion, also promote both the start value and result type and |
--- |
| 2420 |
// then truncate. |
--- |
2420 |
// then truncate. |
--- |
| 2421 |
NewOps[0] = |
0 |
2421 |
NewOps[0] = |
0 |
| 2422 |
DAG.getNode(getExtendForIntVecReduction(N), DL, EltVT, N->getOperand(0)); |
0 |
2422 |
DAG.getNode(getExtendForIntVecReduction(N), DL, EltVT, N->getOperand(0)); |
0 |
| 2423 |
SDValue Reduce = DAG.getNode(N->getOpcode(), DL, EltVT, NewOps); |
0 |
2423 |
SDValue Reduce = DAG.getNode(N->getOpcode(), DL, EltVT, NewOps); |
0 |
| 2424 |
return DAG.getNode(ISD::TRUNCATE, DL, VT, Reduce); |
0 |
2424 |
return DAG.getNode(ISD::TRUNCATE, DL, VT, Reduce); |
0 |
| 2425 |
} |
0 |
2425 |
} |
0 |
| 2426 |
|
--- |
2426 |
|
--- |
| 2427 |
SDValue DAGTypeLegalizer::PromoteIntOp_SET_ROUNDING(SDNode *N) { |
0 |
2427 |
SDValue DAGTypeLegalizer::PromoteIntOp_SET_ROUNDING(SDNode *N) { |
0 |
| 2428 |
SDValue Op = ZExtPromotedInteger(N->getOperand(1)); |
0 |
2428 |
SDValue Op = ZExtPromotedInteger(N->getOperand(1)); |
0 |
| 2429 |
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0); |
0 |
2429 |
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0); |
0 |
| 2430 |
} |
--- |
2430 |
} |
--- |
| 2431 |
|
--- |
2431 |
|
--- |
| 2432 |
SDValue DAGTypeLegalizer::PromoteIntOp_STACKMAP(SDNode *N, unsigned OpNo) { |
0 |
2432 |
SDValue DAGTypeLegalizer::PromoteIntOp_STACKMAP(SDNode *N, unsigned OpNo) { |
0 |
| 2433 |
assert(OpNo > 1); // Because the first two arguments are guaranteed legal. |
0 |
2433 |
assert(OpNo > 1); // Because the first two arguments are guaranteed legal. |
0 |
| 2434 |
SmallVector NewOps(N->ops().begin(), N->ops().end()); |
0 |
2434 |
SmallVector NewOps(N->ops().begin(), N->ops().end()); |
0 |
| 2435 |
SDValue Operand = N->getOperand(OpNo); |
0 |
2435 |
SDValue Operand = N->getOperand(OpNo); |
0 |
| 2436 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Operand.getValueType()); |
0 |
2436 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Operand.getValueType()); |
0 |
| 2437 |
NewOps[OpNo] = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Operand); |
0 |
2437 |
NewOps[OpNo] = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Operand); |
0 |
| 2438 |
return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); |
0 |
2438 |
return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); |
0 |
| 2439 |
} |
0 |
2439 |
} |
0 |
| 2440 |
|
--- |
2440 |
|
--- |
| 2441 |
SDValue DAGTypeLegalizer::PromoteIntOp_PATCHPOINT(SDNode *N, unsigned OpNo) { |
0 |
2441 |
SDValue DAGTypeLegalizer::PromoteIntOp_PATCHPOINT(SDNode *N, unsigned OpNo) { |
0 |
| 2442 |
assert(OpNo >= 7); |
0 |
2442 |
assert(OpNo >= 7); |
0 |
| 2443 |
SmallVector NewOps(N->ops().begin(), N->ops().end()); |
0 |
2443 |
SmallVector NewOps(N->ops().begin(), N->ops().end()); |
0 |
| 2444 |
SDValue Operand = N->getOperand(OpNo); |
0 |
2444 |
SDValue Operand = N->getOperand(OpNo); |
0 |
| 2445 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Operand.getValueType()); |
0 |
2445 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Operand.getValueType()); |
0 |
| 2446 |
NewOps[OpNo] = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Operand); |
0 |
2446 |
NewOps[OpNo] = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Operand); |
0 |
| 2447 |
return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); |
0 |
2447 |
return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); |
0 |
| 2448 |
} |
0 |
2448 |
} |
0 |
| 2449 |
|
--- |
2449 |
|
--- |
| 2450 |
SDValue DAGTypeLegalizer::PromoteIntOp_VP_STRIDED(SDNode *N, unsigned OpNo) { |
0 |
2450 |
SDValue DAGTypeLegalizer::PromoteIntOp_VP_STRIDED(SDNode *N, unsigned OpNo) { |
0 |
| 2451 |
assert((N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) || |
0 |
2451 |
assert((N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) || |
0 |
| 2452 |
(N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4)); |
--- |
2452 |
(N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4)); |
--- |
| 2453 |
|
--- |
2453 |
|
--- |
| 2454 |
SmallVector NewOps(N->op_begin(), N->op_end()); |
0 |
2454 |
SmallVector NewOps(N->op_begin(), N->op_end()); |
0 |
| 2455 |
NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo)); |
0 |
2455 |
NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo)); |
0 |
| 2456 |
|
--- |
2456 |
|
--- |
| 2457 |
return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); |
0 |
2457 |
return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); |
0 |
| 2458 |
} |
0 |
2458 |
} |
0 |
| 2459 |
|
--- |
2459 |
|
--- |
| 2460 |
//===----------------------------------------------------------------------===// |
--- |
2460 |
//===----------------------------------------------------------------------===// |
--- |
| 2461 |
// Integer Result Expansion |
--- |
2461 |
// Integer Result Expansion |
--- |
| 2462 |
//===----------------------------------------------------------------------===// |
--- |
2462 |
//===----------------------------------------------------------------------===// |
--- |
| 2463 |
|
--- |
2463 |
|
--- |
| 2464 |
/// ExpandIntegerResult - This method is called when the specified result of the |
--- |
2464 |
/// ExpandIntegerResult - This method is called when the specified result of the |
--- |
| 2465 |
/// specified node is found to need expansion. At this point, the node may also |
--- |
2465 |
/// specified node is found to need expansion. At this point, the node may also |
--- |
| 2466 |
/// have invalid operands or may have other results that need promotion, we just |
--- |
2466 |
/// have invalid operands or may have other results that need promotion, we just |
--- |
| 2467 |
/// know that (at least) one result needs expansion. |
--- |
2467 |
/// know that (at least) one result needs expansion. |
--- |
| 2468 |
void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { |
0 |
2468 |
void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { |
0 |
| 2469 |
LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG); |
0 |
2469 |
LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG); |
0 |
| 2470 |
dbgs() << "\n"); |
--- |
2470 |
dbgs() << "\n"); |
--- |
| 2471 |
SDValue Lo, Hi; |
0 |
2471 |
SDValue Lo, Hi; |
0 |
| 2472 |
Lo = Hi = SDValue(); |
0 |
2472 |
Lo = Hi = SDValue(); |
0 |
| 2473 |
|
--- |
2473 |
|
--- |
| 2474 |
// See if the target wants to custom expand this node. |
--- |
2474 |
// See if the target wants to custom expand this node. |
--- |
| 2475 |
if (CustomLowerNode(N, N->getValueType(ResNo), true)) |
0 |
2475 |
if (CustomLowerNode(N, N->getValueType(ResNo), true)) |
0 |
| 2476 |
return; |
0 |
2476 |
return; |
0 |
| 2477 |
|
--- |
2477 |
|
--- |
| 2478 |
switch (N->getOpcode()) { |
0 |
2478 |
switch (N->getOpcode()) { |
0 |
| 2479 |
default: |
0 |
2479 |
default: |
0 |
| 2480 |
#ifndef NDEBUG |
--- |
2480 |
#ifndef NDEBUG |
--- |
| 2481 |
dbgs() << "ExpandIntegerResult #" << ResNo << ": "; |
0 |
2481 |
dbgs() << "ExpandIntegerResult #" << ResNo << ": "; |
0 |
| 2482 |
N->dump(&DAG); dbgs() << "\n"; |
0 |
2482 |
N->dump(&DAG); dbgs() << "\n"; |
0 |
| 2483 |
#endif |
--- |
2483 |
#endif |
--- |
| 2484 |
report_fatal_error("Do not know how to expand the result of this " |
0 |
2484 |
report_fatal_error("Do not know how to expand the result of this " |
0 |
| 2485 |
"operator!"); |
--- |
2485 |
"operator!"); |
--- |
| 2486 |
|
--- |
2486 |
|
--- |
| 2487 |
case ISD::ARITH_FENCE: SplitRes_ARITH_FENCE(N, Lo, Hi); break; |
0 |
2487 |
case ISD::ARITH_FENCE: SplitRes_ARITH_FENCE(N, Lo, Hi); break; |
0 |
| 2488 |
case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; |
0 |
2488 |
case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; |
0 |
| 2489 |
case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break; |
0 |
2489 |
case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break; |
0 |
| 2490 |
case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; |
0 |
2490 |
case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; |
0 |
| 2491 |
case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; |
0 |
2491 |
case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; |
0 |
| 2492 |
case ISD::FREEZE: SplitRes_FREEZE(N, Lo, Hi); break; |
0 |
2492 |
case ISD::FREEZE: SplitRes_FREEZE(N, Lo, Hi); break; |
0 |
| 2493 |
|
--- |
2493 |
|
--- |
| 2494 |
case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break; |
0 |
2494 |
case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break; |
0 |
| 2495 |
case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break; |
0 |
2495 |
case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break; |
0 |
| 2496 |
case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break; |
0 |
2496 |
case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break; |
0 |
| 2497 |
case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break; |
0 |
2497 |
case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break; |
0 |
| 2498 |
case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break; |
0 |
2498 |
case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break; |
0 |
| 2499 |
|
--- |
2499 |
|
--- |
| 2500 |
case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break; |
0 |
2500 |
case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break; |
0 |
| 2501 |
case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break; |
0 |
2501 |
case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break; |
0 |
| 2502 |
case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break; |
0 |
2502 |
case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break; |
0 |
| 2503 |
case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break; |
0 |
2503 |
case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break; |
0 |
| 2504 |
case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break; |
0 |
2504 |
case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break; |
0 |
| 2505 |
case ISD::PARITY: ExpandIntRes_PARITY(N, Lo, Hi); break; |
0 |
2505 |
case ISD::PARITY: ExpandIntRes_PARITY(N, Lo, Hi); break; |
0 |
| 2506 |
case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break; |
0 |
2506 |
case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break; |
0 |
| 2507 |
case ISD::ABS: ExpandIntRes_ABS(N, Lo, Hi); break; |
0 |
2507 |
case ISD::ABS: ExpandIntRes_ABS(N, Lo, Hi); break; |
0 |
| 2508 |
case ISD::CTLZ_ZERO_UNDEF: |
0 |
2508 |
case ISD::CTLZ_ZERO_UNDEF: |
0 |
| 2509 |
case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break; |
0 |
2509 |
case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break; |
0 |
| 2510 |
case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break; |
0 |
2510 |
case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break; |
0 |
| 2511 |
case ISD::CTTZ_ZERO_UNDEF: |
0 |
2511 |
case ISD::CTTZ_ZERO_UNDEF: |
0 |
| 2512 |
case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break; |
0 |
2512 |
case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break; |
0 |
| 2513 |
case ISD::GET_ROUNDING:ExpandIntRes_GET_ROUNDING(N, Lo, Hi); break; |
0 |
2513 |
case ISD::GET_ROUNDING:ExpandIntRes_GET_ROUNDING(N, Lo, Hi); break; |
0 |
| 2514 |
case ISD::STRICT_FP_TO_SINT: |
0 |
2514 |
case ISD::STRICT_FP_TO_SINT: |
0 |
| 2515 |
case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break; |
0 |
2515 |
case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break; |
0 |
| 2516 |
case ISD::STRICT_FP_TO_UINT: |
0 |
2516 |
case ISD::STRICT_FP_TO_UINT: |
0 |
| 2517 |
case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break; |
0 |
2517 |
case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break; |
0 |
| 2518 |
case ISD::FP_TO_SINT_SAT: |
0 |
2518 |
case ISD::FP_TO_SINT_SAT: |
0 |
| 2519 |
case ISD::FP_TO_UINT_SAT: ExpandIntRes_FP_TO_XINT_SAT(N, Lo, Hi); break; |
0 |
2519 |
case ISD::FP_TO_UINT_SAT: ExpandIntRes_FP_TO_XINT_SAT(N, Lo, Hi); break; |
0 |
| 2520 |
case ISD::STRICT_LROUND: |
0 |
2520 |
case ISD::STRICT_LROUND: |
0 |
| 2521 |
case ISD::STRICT_LRINT: |
--- |
2521 |
case ISD::STRICT_LRINT: |
--- |
| 2522 |
case ISD::LROUND: |
--- |
2522 |
case ISD::LROUND: |
--- |
| 2523 |
case ISD::LRINT: |
--- |
2523 |
case ISD::LRINT: |
--- |
| 2524 |
case ISD::STRICT_LLROUND: |
--- |
2524 |
case ISD::STRICT_LLROUND: |
--- |
| 2525 |
case ISD::STRICT_LLRINT: |
--- |
2525 |
case ISD::STRICT_LLRINT: |
--- |
| 2526 |
case ISD::LLROUND: |
--- |
2526 |
case ISD::LLROUND: |
--- |
| 2527 |
case ISD::LLRINT: ExpandIntRes_XROUND_XRINT(N, Lo, Hi); break; |
0 |
2527 |
case ISD::LLRINT: ExpandIntRes_XROUND_XRINT(N, Lo, Hi); break; |
0 |
| 2528 |
case ISD::LOAD: ExpandIntRes_LOAD(cast(N), Lo, Hi); break; |
0 |
2528 |
case ISD::LOAD: ExpandIntRes_LOAD(cast(N), Lo, Hi); break; |
0 |
| 2529 |
case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break; |
0 |
2529 |
case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break; |
0 |
| 2530 |
case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break; |
0 |
2530 |
case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break; |
0 |
| 2531 |
case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break; |
0 |
2531 |
case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break; |
0 |
| 2532 |
case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break; |
0 |
2532 |
case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break; |
0 |
| 2533 |
case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break; |
0 |
2533 |
case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break; |
0 |
| 2534 |
case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break; |
0 |
2534 |
case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break; |
0 |
| 2535 |
case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break; |
0 |
2535 |
case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break; |
0 |
| 2536 |
case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break; |
0 |
2536 |
case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break; |
0 |
| 2537 |
case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break; |
0 |
2537 |
case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break; |
0 |
| 2538 |
case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break; |
0 |
2538 |
case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break; |
0 |
| 2539 |
case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break; |
0 |
2539 |
case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break; |
0 |
| 2540 |
|
--- |
2540 |
|
--- |
| 2541 |
case ISD::ATOMIC_LOAD_ADD: |
0 |
2541 |
case ISD::ATOMIC_LOAD_ADD: |
0 |
| 2542 |
case ISD::ATOMIC_LOAD_SUB: |
--- |
2542 |
case ISD::ATOMIC_LOAD_SUB: |
--- |
| 2543 |
case ISD::ATOMIC_LOAD_AND: |
--- |
2543 |
case ISD::ATOMIC_LOAD_AND: |
--- |
| 2544 |
case ISD::ATOMIC_LOAD_CLR: |
--- |
2544 |
case ISD::ATOMIC_LOAD_CLR: |
--- |
| 2545 |
case ISD::ATOMIC_LOAD_OR: |
--- |
2545 |
case ISD::ATOMIC_LOAD_OR: |
--- |
| 2546 |
case ISD::ATOMIC_LOAD_XOR: |
--- |
2546 |
case ISD::ATOMIC_LOAD_XOR: |
--- |
| 2547 |
case ISD::ATOMIC_LOAD_NAND: |
--- |
2547 |
case ISD::ATOMIC_LOAD_NAND: |
--- |
| 2548 |
case ISD::ATOMIC_LOAD_MIN: |
--- |
2548 |
case ISD::ATOMIC_LOAD_MIN: |
--- |
| 2549 |
case ISD::ATOMIC_LOAD_MAX: |
--- |
2549 |
case ISD::ATOMIC_LOAD_MAX: |
--- |
| 2550 |
case ISD::ATOMIC_LOAD_UMIN: |
--- |
2550 |
case ISD::ATOMIC_LOAD_UMIN: |
--- |
| 2551 |
case ISD::ATOMIC_LOAD_UMAX: |
--- |
2551 |
case ISD::ATOMIC_LOAD_UMAX: |
--- |
| 2552 |
case ISD::ATOMIC_SWAP: |
--- |
2552 |
case ISD::ATOMIC_SWAP: |
--- |
| 2553 |
case ISD::ATOMIC_CMP_SWAP: { |
--- |
2553 |
case ISD::ATOMIC_CMP_SWAP: { |
--- |
| 2554 |
std::pair Tmp = ExpandAtomic(N); |
0 |
2554 |
std::pair Tmp = ExpandAtomic(N); |
0 |
| 2555 |
SplitInteger(Tmp.first, Lo, Hi); |
0 |
2555 |
SplitInteger(Tmp.first, Lo, Hi); |
0 |
| 2556 |
ReplaceValueWith(SDValue(N, 1), Tmp.second); |
0 |
2556 |
ReplaceValueWith(SDValue(N, 1), Tmp.second); |
0 |
| 2557 |
break; |
0 |
2557 |
break; |
0 |
| 2558 |
} |
--- |
2558 |
} |
--- |
| 2559 |
case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: { |
0 |
2559 |
case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: { |
0 |
| 2560 |
AtomicSDNode *AN = cast(N); |
0 |
2560 |
AtomicSDNode *AN = cast(N); |
0 |
| 2561 |
SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other); |
0 |
2561 |
SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other); |
0 |
| 2562 |
SDValue Tmp = DAG.getAtomicCmpSwap( |
0 |
2562 |
SDValue Tmp = DAG.getAtomicCmpSwap( |
0 |
| 2563 |
ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs, |
0 |
2563 |
ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs, |
0 |
| 2564 |
N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3), |
0 |
2564 |
N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3), |
0 |
| 2565 |
AN->getMemOperand()); |
--- |
2565 |
AN->getMemOperand()); |
--- |
| 2566 |
|
--- |
2566 |
|
--- |
| 2567 |
// Expanding to the strong ATOMIC_CMP_SWAP node means we can determine |
--- |
2567 |
// Expanding to the strong ATOMIC_CMP_SWAP node means we can determine |
--- |
| 2568 |
// success simply by comparing the loaded value against the ingoing |
--- |
2568 |
// success simply by comparing the loaded value against the ingoing |
--- |
| 2569 |
// comparison. |
--- |
2569 |
// comparison. |
--- |
| 2570 |
SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp, |
0 |
2570 |
SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp, |
0 |
| 2571 |
N->getOperand(2), ISD::SETEQ); |
0 |
2571 |
N->getOperand(2), ISD::SETEQ); |
0 |
| 2572 |
|
--- |
2572 |
|
--- |
| 2573 |
SplitInteger(Tmp, Lo, Hi); |
0 |
2573 |
SplitInteger(Tmp, Lo, Hi); |
0 |
| 2574 |
ReplaceValueWith(SDValue(N, 1), Success); |
0 |
2574 |
ReplaceValueWith(SDValue(N, 1), Success); |
0 |
| 2575 |
ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1)); |
0 |
2575 |
ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1)); |
0 |
| 2576 |
break; |
0 |
2576 |
break; |
0 |
| 2577 |
} |
--- |
2577 |
} |
--- |
| 2578 |
|
--- |
2578 |
|
--- |
| 2579 |
case ISD::AND: |
0 |
2579 |
case ISD::AND: |
0 |
| 2580 |
case ISD::OR: |
--- |
2580 |
case ISD::OR: |
--- |
| 2581 |
case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break; |
0 |
2581 |
case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break; |
0 |
| 2582 |
|
--- |
2582 |
|
--- |
| 2583 |
case ISD::UMAX: |
0 |
2583 |
case ISD::UMAX: |
0 |
| 2584 |
case ISD::SMAX: |
--- |
2584 |
case ISD::SMAX: |
--- |
| 2585 |
case ISD::UMIN: |
--- |
2585 |
case ISD::UMIN: |
--- |
| 2586 |
case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break; |
0 |
2586 |
case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break; |
0 |
| 2587 |
|
--- |
2587 |
|
--- |
| 2588 |
case ISD::ADD: |
0 |
2588 |
case ISD::ADD: |
0 |
| 2589 |
case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break; |
0 |
2589 |
case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break; |
0 |
| 2590 |
|
--- |
2590 |
|
--- |
| 2591 |
case ISD::ADDC: |
0 |
2591 |
case ISD::ADDC: |
0 |
| 2592 |
case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break; |
0 |
2592 |
case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break; |
0 |
| 2593 |
|
--- |
2593 |
|
--- |
| 2594 |
case ISD::ADDE: |
0 |
2594 |
case ISD::ADDE: |
0 |
| 2595 |
case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break; |
0 |
2595 |
case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break; |
0 |
| 2596 |
|
--- |
2596 |
|
--- |
| 2597 |
case ISD::UADDO_CARRY: |
0 |
2597 |
case ISD::UADDO_CARRY: |
0 |
| 2598 |
case ISD::USUBO_CARRY: ExpandIntRes_UADDSUBO_CARRY(N, Lo, Hi); break; |
0 |
2598 |
case ISD::USUBO_CARRY: ExpandIntRes_UADDSUBO_CARRY(N, Lo, Hi); break; |
0 |
| 2599 |
|
--- |
2599 |
|
--- |
| 2600 |
case ISD::SADDO_CARRY: |
0 |
2600 |
case ISD::SADDO_CARRY: |
0 |
| 2601 |
case ISD::SSUBO_CARRY: ExpandIntRes_SADDSUBO_CARRY(N, Lo, Hi); break; |
0 |
2601 |
case ISD::SSUBO_CARRY: ExpandIntRes_SADDSUBO_CARRY(N, Lo, Hi); break; |
0 |
| 2602 |
|
--- |
2602 |
|
--- |
| 2603 |
case ISD::SHL: |
0 |
2603 |
case ISD::SHL: |
0 |
| 2604 |
case ISD::SRA: |
--- |
2604 |
case ISD::SRA: |
--- |
| 2605 |
case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break; |
0 |
2605 |
case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break; |
0 |
| 2606 |
|
--- |
2606 |
|
--- |
| 2607 |
case ISD::SADDO: |
0 |
2607 |
case ISD::SADDO: |
0 |
| 2608 |
case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break; |
0 |
2608 |
case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break; |
0 |
| 2609 |
case ISD::UADDO: |
0 |
2609 |
case ISD::UADDO: |
0 |
| 2610 |
case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break; |
0 |
2610 |
case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break; |
0 |
| 2611 |
case ISD::UMULO: |
0 |
2611 |
case ISD::UMULO: |
0 |
| 2612 |
case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break; |
0 |
2612 |
case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break; |
0 |
| 2613 |
|
--- |
2613 |
|
--- |
| 2614 |
case ISD::SADDSAT: |
0 |
2614 |
case ISD::SADDSAT: |
0 |
| 2615 |
case ISD::UADDSAT: |
--- |
2615 |
case ISD::UADDSAT: |
--- |
| 2616 |
case ISD::SSUBSAT: |
--- |
2616 |
case ISD::SSUBSAT: |
--- |
| 2617 |
case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break; |
0 |
2617 |
case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break; |
0 |
| 2618 |
|
--- |
2618 |
|
--- |
| 2619 |
case ISD::SSHLSAT: |
0 |
2619 |
case ISD::SSHLSAT: |
0 |
| 2620 |
case ISD::USHLSAT: ExpandIntRes_SHLSAT(N, Lo, Hi); break; |
0 |
2620 |
case ISD::USHLSAT: ExpandIntRes_SHLSAT(N, Lo, Hi); break; |
0 |
| 2621 |
|
--- |
2621 |
|
--- |
| 2622 |
case ISD::SMULFIX: |
0 |
2622 |
case ISD::SMULFIX: |
0 |
| 2623 |
case ISD::SMULFIXSAT: |
--- |
2623 |
case ISD::SMULFIXSAT: |
--- |
| 2624 |
case ISD::UMULFIX: |
--- |
2624 |
case ISD::UMULFIX: |
--- |
| 2625 |
case ISD::UMULFIXSAT: ExpandIntRes_MULFIX(N, Lo, Hi); break; |
0 |
2625 |
case ISD::UMULFIXSAT: ExpandIntRes_MULFIX(N, Lo, Hi); break; |
0 |
| 2626 |
|
--- |
2626 |
|
--- |
| 2627 |
case ISD::SDIVFIX: |
0 |
2627 |
case ISD::SDIVFIX: |
0 |
| 2628 |
case ISD::SDIVFIXSAT: |
--- |
2628 |
case ISD::SDIVFIXSAT: |
--- |
| 2629 |
case ISD::UDIVFIX: |
--- |
2629 |
case ISD::UDIVFIX: |
--- |
| 2630 |
case ISD::UDIVFIXSAT: ExpandIntRes_DIVFIX(N, Lo, Hi); break; |
0 |
2630 |
case ISD::UDIVFIXSAT: ExpandIntRes_DIVFIX(N, Lo, Hi); break; |
0 |
| 2631 |
|
--- |
2631 |
|
--- |
| 2632 |
case ISD::VECREDUCE_ADD: |
0 |
2632 |
case ISD::VECREDUCE_ADD: |
0 |
| 2633 |
case ISD::VECREDUCE_MUL: |
--- |
2633 |
case ISD::VECREDUCE_MUL: |
--- |
| 2634 |
case ISD::VECREDUCE_AND: |
--- |
2634 |
case ISD::VECREDUCE_AND: |
--- |
| 2635 |
case ISD::VECREDUCE_OR: |
--- |
2635 |
case ISD::VECREDUCE_OR: |
--- |
| 2636 |
case ISD::VECREDUCE_XOR: |
--- |
2636 |
case ISD::VECREDUCE_XOR: |
--- |
| 2637 |
case ISD::VECREDUCE_SMAX: |
--- |
2637 |
case ISD::VECREDUCE_SMAX: |
--- |
| 2638 |
case ISD::VECREDUCE_SMIN: |
--- |
2638 |
case ISD::VECREDUCE_SMIN: |
--- |
| 2639 |
case ISD::VECREDUCE_UMAX: |
--- |
2639 |
case ISD::VECREDUCE_UMAX: |
--- |
| 2640 |
case ISD::VECREDUCE_UMIN: ExpandIntRes_VECREDUCE(N, Lo, Hi); break; |
0 |
2640 |
case ISD::VECREDUCE_UMIN: ExpandIntRes_VECREDUCE(N, Lo, Hi); break; |
0 |
| 2641 |
|
--- |
2641 |
|
--- |
| 2642 |
case ISD::ROTL: |
0 |
2642 |
case ISD::ROTL: |
0 |
| 2643 |
case ISD::ROTR: |
--- |
2643 |
case ISD::ROTR: |
--- |
| 2644 |
ExpandIntRes_Rotate(N, Lo, Hi); |
0 |
2644 |
ExpandIntRes_Rotate(N, Lo, Hi); |
0 |
| 2645 |
break; |
0 |
2645 |
break; |
0 |
| 2646 |
|
--- |
2646 |
|
--- |
| 2647 |
case ISD::FSHL: |
0 |
2647 |
case ISD::FSHL: |
0 |
| 2648 |
case ISD::FSHR: |
--- |
2648 |
case ISD::FSHR: |
--- |
| 2649 |
ExpandIntRes_FunnelShift(N, Lo, Hi); |
0 |
2649 |
ExpandIntRes_FunnelShift(N, Lo, Hi); |
0 |
| 2650 |
break; |
0 |
2650 |
break; |
0 |
| 2651 |
|
--- |
2651 |
|
--- |
| 2652 |
case ISD::VSCALE: |
0 |
2652 |
case ISD::VSCALE: |
0 |
| 2653 |
ExpandIntRes_VSCALE(N, Lo, Hi); |
0 |
2653 |
ExpandIntRes_VSCALE(N, Lo, Hi); |
0 |
| 2654 |
break; |
0 |
2654 |
break; |
0 |
| 2655 |
} |
--- |
2655 |
} |
--- |
| 2656 |
|
--- |
2656 |
|
--- |
| 2657 |
// If Lo/Hi is null, the sub-method took care of registering results etc. |
--- |
2657 |
// If Lo/Hi is null, the sub-method took care of registering results etc. |
--- |
| 2658 |
if (Lo.getNode()) |
0 |
2658 |
if (Lo.getNode()) |
0 |
| 2659 |
SetExpandedInteger(SDValue(N, ResNo), Lo, Hi); |
0 |
2659 |
SetExpandedInteger(SDValue(N, ResNo), Lo, Hi); |
0 |
| 2660 |
} |
--- |
2660 |
} |
--- |
| 2661 |
|
--- |
2661 |
|
--- |
| 2662 |
/// Lower an atomic node to the appropriate builtin call. |
--- |
2662 |
/// Lower an atomic node to the appropriate builtin call. |
--- |
| 2663 |
std::pair DAGTypeLegalizer::ExpandAtomic(SDNode *Node) { |
0 |
2663 |
std::pair DAGTypeLegalizer::ExpandAtomic(SDNode *Node) { |
0 |
| 2664 |
unsigned Opc = Node->getOpcode(); |
0 |
2664 |
unsigned Opc = Node->getOpcode(); |
0 |
| 2665 |
MVT VT = cast(Node)->getMemoryVT().getSimpleVT(); |
0 |
2665 |
MVT VT = cast(Node)->getMemoryVT().getSimpleVT(); |
0 |
| 2666 |
AtomicOrdering order = cast(Node)->getMergedOrdering(); |
0 |
2666 |
AtomicOrdering order = cast(Node)->getMergedOrdering(); |
0 |
| 2667 |
// Lower to outline atomic libcall if outline atomics enabled, |
--- |
2667 |
// Lower to outline atomic libcall if outline atomics enabled, |
--- |
| 2668 |
// or to sync libcall otherwise |
--- |
2668 |
// or to sync libcall otherwise |
--- |
| 2669 |
RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, order, VT); |
0 |
2669 |
RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, order, VT); |
0 |
| 2670 |
EVT RetVT = Node->getValueType(0); |
0 |
2670 |
EVT RetVT = Node->getValueType(0); |
0 |
| 2671 |
TargetLowering::MakeLibCallOptions CallOptions; |
0 |
2671 |
TargetLowering::MakeLibCallOptions CallOptions; |
0 |
| 2672 |
SmallVector Ops; |
0 |
2672 |
SmallVector Ops; |
0 |
| 2673 |
if (TLI.getLibcallName(LC)) { |
0 |
2673 |
if (TLI.getLibcallName(LC)) { |
0 |
| 2674 |
Ops.append(Node->op_begin() + 2, Node->op_end()); |
0 |
2674 |
Ops.append(Node->op_begin() + 2, Node->op_end()); |
0 |
| 2675 |
Ops.push_back(Node->getOperand(1)); |
0 |
2675 |
Ops.push_back(Node->getOperand(1)); |
0 |
| 2676 |
} else { |
--- |
2676 |
} else { |
--- |
| 2677 |
LC = RTLIB::getSYNC(Opc, VT); |
0 |
2677 |
LC = RTLIB::getSYNC(Opc, VT); |
0 |
| 2678 |
assert(LC != RTLIB::UNKNOWN_LIBCALL && |
0 |
2678 |
assert(LC != RTLIB::UNKNOWN_LIBCALL && |
0 |
| 2679 |
"Unexpected atomic op or value type!"); |
--- |
2679 |
"Unexpected atomic op or value type!"); |
--- |
| 2680 |
Ops.append(Node->op_begin() + 1, Node->op_end()); |
0 |
2680 |
Ops.append(Node->op_begin() + 1, Node->op_end()); |
0 |
| 2681 |
} |
--- |
2681 |
} |
--- |
| 2682 |
return TLI.makeLibCall(DAG, LC, RetVT, Ops, CallOptions, SDLoc(Node), |
0 |
2682 |
return TLI.makeLibCall(DAG, LC, RetVT, Ops, CallOptions, SDLoc(Node), |
0 |
| 2683 |
Node->getOperand(0)); |
0 |
2683 |
Node->getOperand(0)); |
0 |
| 2684 |
} |
0 |
2684 |
} |
0 |
| 2685 |
|
--- |
2685 |
|
--- |
| 2686 |
/// N is a shift by a value that needs to be expanded, |
--- |
2686 |
/// N is a shift by a value that needs to be expanded, |
--- |
| 2687 |
/// and the shift amount is a constant 'Amt'. Expand the operation. |
--- |
2687 |
/// and the shift amount is a constant 'Amt'. Expand the operation. |
--- |
| 2688 |
void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt, |
0 |
2688 |
void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt, |
0 |
| 2689 |
SDValue &Lo, SDValue &Hi) { |
--- |
2689 |
SDValue &Lo, SDValue &Hi) { |
--- |
| 2690 |
SDLoc DL(N); |
0 |
2690 |
SDLoc DL(N); |
0 |
| 2691 |
// Expand the incoming operand to be shifted, so that we have its parts |
--- |
2691 |
// Expand the incoming operand to be shifted, so that we have its parts |
--- |
| 2692 |
SDValue InL, InH; |
0 |
2692 |
SDValue InL, InH; |
0 |
| 2693 |
GetExpandedInteger(N->getOperand(0), InL, InH); |
0 |
2693 |
GetExpandedInteger(N->getOperand(0), InL, InH); |
0 |
| 2694 |
|
--- |
2694 |
|
--- |
| 2695 |
// Though Amt shouldn't usually be 0, it's possible. E.g. when legalization |
--- |
2695 |
// Though Amt shouldn't usually be 0, it's possible. E.g. when legalization |
--- |
| 2696 |
// splitted a vector shift, like this: SHL <0, 2>. |
--- |
2696 |
// splitted a vector shift, like this: SHL <0, 2>. |
--- |
| 2697 |
if (!Amt) { |
0 |
2697 |
if (!Amt) { |
0 |
| 2698 |
Lo = InL; |
0 |
2698 |
Lo = InL; |
0 |
| 2699 |
Hi = InH; |
0 |
2699 |
Hi = InH; |
0 |
| 2700 |
return; |
0 |
2700 |
return; |
0 |
| 2701 |
} |
--- |
2701 |
} |
--- |
| 2702 |
|
--- |
2702 |
|
--- |
| 2703 |
EVT NVT = InL.getValueType(); |
0 |
2703 |
EVT NVT = InL.getValueType(); |
0 |
| 2704 |
unsigned VTBits = N->getValueType(0).getSizeInBits(); |
0 |
2704 |
unsigned VTBits = N->getValueType(0).getSizeInBits(); |
0 |
| 2705 |
unsigned NVTBits = NVT.getSizeInBits(); |
0 |
2705 |
unsigned NVTBits = NVT.getSizeInBits(); |
0 |
| 2706 |
EVT ShTy = N->getOperand(1).getValueType(); |
0 |
2706 |
EVT ShTy = N->getOperand(1).getValueType(); |
0 |
| 2707 |
|
--- |
2707 |
|
--- |
| 2708 |
if (N->getOpcode() == ISD::SHL) { |
0 |
2708 |
if (N->getOpcode() == ISD::SHL) { |
0 |
| 2709 |
if (Amt.uge(VTBits)) { |
0 |
2709 |
if (Amt.uge(VTBits)) { |
0 |
| 2710 |
Lo = Hi = DAG.getConstant(0, DL, NVT); |
0 |
2710 |
Lo = Hi = DAG.getConstant(0, DL, NVT); |
0 |
| 2711 |
} else if (Amt.ugt(NVTBits)) { |
0 |
2711 |
} else if (Amt.ugt(NVTBits)) { |
0 |
| 2712 |
Lo = DAG.getConstant(0, DL, NVT); |
0 |
2712 |
Lo = DAG.getConstant(0, DL, NVT); |
0 |
| 2713 |
Hi = DAG.getNode(ISD::SHL, DL, |
0 |
2713 |
Hi = DAG.getNode(ISD::SHL, DL, |
0 |
| 2714 |
NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy)); |
0 |
2714 |
NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy)); |
0 |
| 2715 |
} else if (Amt == NVTBits) { |
0 |
2715 |
} else if (Amt == NVTBits) { |
0 |
| 2716 |
Lo = DAG.getConstant(0, DL, NVT); |
0 |
2716 |
Lo = DAG.getConstant(0, DL, NVT); |
0 |
| 2717 |
Hi = InL; |
0 |
2717 |
Hi = InL; |
0 |
| 2718 |
} else { |
--- |
2718 |
} else { |
--- |
| 2719 |
Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy)); |
0 |
2719 |
Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy)); |
0 |
| 2720 |
Hi = DAG.getNode(ISD::OR, DL, NVT, |
0 |
2720 |
Hi = DAG.getNode(ISD::OR, DL, NVT, |
0 |
| 2721 |
DAG.getNode(ISD::SHL, DL, NVT, InH, |
0 |
2721 |
DAG.getNode(ISD::SHL, DL, NVT, InH, |
0 |
| 2722 |
DAG.getConstant(Amt, DL, ShTy)), |
0 |
2722 |
DAG.getConstant(Amt, DL, ShTy)), |
0 |
| 2723 |
DAG.getNode(ISD::SRL, DL, NVT, InL, |
0 |
2723 |
DAG.getNode(ISD::SRL, DL, NVT, InL, |
0 |
| 2724 |
DAG.getConstant(-Amt + NVTBits, DL, ShTy))); |
0 |
2724 |
DAG.getConstant(-Amt + NVTBits, DL, ShTy))); |
0 |
| 2725 |
} |
--- |
2725 |
} |
--- |
| 2726 |
return; |
0 |
2726 |
return; |
0 |
| 2727 |
} |
--- |
2727 |
} |
--- |
| 2728 |
|
--- |
2728 |
|
--- |
| 2729 |
if (N->getOpcode() == ISD::SRL) { |
0 |
2729 |
if (N->getOpcode() == ISD::SRL) { |
0 |
| 2730 |
if (Amt.uge(VTBits)) { |
0 |
2730 |
if (Amt.uge(VTBits)) { |
0 |
| 2731 |
Lo = Hi = DAG.getConstant(0, DL, NVT); |
0 |
2731 |
Lo = Hi = DAG.getConstant(0, DL, NVT); |
0 |
| 2732 |
} else if (Amt.ugt(NVTBits)) { |
0 |
2732 |
} else if (Amt.ugt(NVTBits)) { |
0 |
| 2733 |
Lo = DAG.getNode(ISD::SRL, DL, |
0 |
2733 |
Lo = DAG.getNode(ISD::SRL, DL, |
0 |
| 2734 |
NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy)); |
0 |
2734 |
NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy)); |
0 |
| 2735 |
Hi = DAG.getConstant(0, DL, NVT); |
0 |
2735 |
Hi = DAG.getConstant(0, DL, NVT); |
0 |
| 2736 |
} else if (Amt == NVTBits) { |
0 |
2736 |
} else if (Amt == NVTBits) { |
0 |
| 2737 |
Lo = InH; |
0 |
2737 |
Lo = InH; |
0 |
| 2738 |
Hi = DAG.getConstant(0, DL, NVT); |
0 |
2738 |
Hi = DAG.getConstant(0, DL, NVT); |
0 |
| 2739 |
} else { |
--- |
2739 |
} else { |
--- |
| 2740 |
Lo = DAG.getNode(ISD::OR, DL, NVT, |
0 |
2740 |
Lo = DAG.getNode(ISD::OR, DL, NVT, |
0 |
| 2741 |
DAG.getNode(ISD::SRL, DL, NVT, InL, |
0 |
2741 |
DAG.getNode(ISD::SRL, DL, NVT, InL, |
0 |
| 2742 |
DAG.getConstant(Amt, DL, ShTy)), |
0 |
2742 |
DAG.getConstant(Amt, DL, ShTy)), |
0 |
| 2743 |
DAG.getNode(ISD::SHL, DL, NVT, InH, |
0 |
2743 |
DAG.getNode(ISD::SHL, DL, NVT, InH, |
0 |
| 2744 |
DAG.getConstant(-Amt + NVTBits, DL, ShTy))); |
0 |
2744 |
DAG.getConstant(-Amt + NVTBits, DL, ShTy))); |
0 |
| 2745 |
Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy)); |
0 |
2745 |
Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy)); |
0 |
| 2746 |
} |
--- |
2746 |
} |
--- |
| 2747 |
return; |
0 |
2747 |
return; |
0 |
| 2748 |
} |
--- |
2748 |
} |
--- |
| 2749 |
|
--- |
2749 |
|
--- |
| 2750 |
assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); |
0 |
2750 |
assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); |
0 |
| 2751 |
if (Amt.uge(VTBits)) { |
0 |
2751 |
if (Amt.uge(VTBits)) { |
0 |
| 2752 |
Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, |
0 |
2752 |
Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, |
0 |
| 2753 |
DAG.getConstant(NVTBits - 1, DL, ShTy)); |
0 |
2753 |
DAG.getConstant(NVTBits - 1, DL, ShTy)); |
0 |
| 2754 |
} else if (Amt.ugt(NVTBits)) { |
0 |
2754 |
} else if (Amt.ugt(NVTBits)) { |
0 |
| 2755 |
Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, |
0 |
2755 |
Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, |
0 |
| 2756 |
DAG.getConstant(Amt - NVTBits, DL, ShTy)); |
0 |
2756 |
DAG.getConstant(Amt - NVTBits, DL, ShTy)); |
0 |
| 2757 |
Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, |
0 |
2757 |
Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, |
0 |
| 2758 |
DAG.getConstant(NVTBits - 1, DL, ShTy)); |
0 |
2758 |
DAG.getConstant(NVTBits - 1, DL, ShTy)); |
0 |
| 2759 |
} else if (Amt == NVTBits) { |
0 |
2759 |
} else if (Amt == NVTBits) { |
0 |
| 2760 |
Lo = InH; |
0 |
2760 |
Lo = InH; |
0 |
| 2761 |
Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, |
0 |
2761 |
Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, |
0 |
| 2762 |
DAG.getConstant(NVTBits - 1, DL, ShTy)); |
0 |
2762 |
DAG.getConstant(NVTBits - 1, DL, ShTy)); |
0 |
| 2763 |
} else { |
--- |
2763 |
} else { |
--- |
| 2764 |
Lo = DAG.getNode(ISD::OR, DL, NVT, |
0 |
2764 |
Lo = DAG.getNode(ISD::OR, DL, NVT, |
0 |
| 2765 |
DAG.getNode(ISD::SRL, DL, NVT, InL, |
0 |
2765 |
DAG.getNode(ISD::SRL, DL, NVT, InL, |
0 |
| 2766 |
DAG.getConstant(Amt, DL, ShTy)), |
0 |
2766 |
DAG.getConstant(Amt, DL, ShTy)), |
0 |
| 2767 |
DAG.getNode(ISD::SHL, DL, NVT, InH, |
0 |
2767 |
DAG.getNode(ISD::SHL, DL, NVT, InH, |
0 |
| 2768 |
DAG.getConstant(-Amt + NVTBits, DL, ShTy))); |
0 |
2768 |
DAG.getConstant(-Amt + NVTBits, DL, ShTy))); |
0 |
| 2769 |
Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy)); |
0 |
2769 |
Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy)); |
0 |
| 2770 |
} |
--- |
2770 |
} |
--- |
| 2771 |
} |
0 |
2771 |
} |
0 |
| 2772 |
|
--- |
2772 |
|
--- |
| 2773 |
/// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify |
--- |
2773 |
/// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify |
--- |
| 2774 |
/// this shift based on knowledge of the high bit of the shift amount. If we |
--- |
2774 |
/// this shift based on knowledge of the high bit of the shift amount. If we |
--- |
| 2775 |
/// can tell this, we know that it is >= 32 or < 32, without knowing the actual |
--- |
2775 |
/// can tell this, we know that it is >= 32 or < 32, without knowing the actual |
--- |
| 2776 |
/// shift amount. |
--- |
2776 |
/// shift amount. |
--- |
| 2777 |
bool DAGTypeLegalizer:: |
0 |
2777 |
bool DAGTypeLegalizer:: |
0 |
| 2778 |
ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { |
--- |
2778 |
ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { |
--- |
| 2779 |
SDValue Amt = N->getOperand(1); |
0 |
2779 |
SDValue Amt = N->getOperand(1); |
0 |
| 2780 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
2780 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
| 2781 |
EVT ShTy = Amt.getValueType(); |
0 |
2781 |
EVT ShTy = Amt.getValueType(); |
0 |
| 2782 |
unsigned ShBits = ShTy.getScalarSizeInBits(); |
0 |
2782 |
unsigned ShBits = ShTy.getScalarSizeInBits(); |
0 |
| 2783 |
unsigned NVTBits = NVT.getScalarSizeInBits(); |
0 |
2783 |
unsigned NVTBits = NVT.getScalarSizeInBits(); |
0 |
| 2784 |
assert(isPowerOf2_32(NVTBits) && |
0 |
2784 |
assert(isPowerOf2_32(NVTBits) && |
0 |
| 2785 |
"Expanded integer type size not a power of two!"); |
--- |
2785 |
"Expanded integer type size not a power of two!"); |
--- |
| 2786 |
SDLoc dl(N); |
0 |
2786 |
SDLoc dl(N); |
0 |
| 2787 |
|
--- |
2787 |
|
--- |
| 2788 |
APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits)); |
0 |
2788 |
APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits)); |
0 |
| 2789 |
KnownBits Known = DAG.computeKnownBits(N->getOperand(1)); |
0 |
2789 |
KnownBits Known = DAG.computeKnownBits(N->getOperand(1)); |
0 |
| 2790 |
|
--- |
2790 |
|
--- |
| 2791 |
// If we don't know anything about the high bits, exit. |
--- |
2791 |
// If we don't know anything about the high bits, exit. |
--- |
| 2792 |
if (((Known.Zero|Known.One) & HighBitMask) == 0) |
0 |
2792 |
if (((Known.Zero|Known.One) & HighBitMask) == 0) |
0 |
| 2793 |
return false; |
0 |
2793 |
return false; |
0 |
| 2794 |
|
--- |
2794 |
|
--- |
| 2795 |
// Get the incoming operand to be shifted. |
--- |
2795 |
// Get the incoming operand to be shifted. |
--- |
| 2796 |
SDValue InL, InH; |
0 |
2796 |
SDValue InL, InH; |
0 |
| 2797 |
GetExpandedInteger(N->getOperand(0), InL, InH); |
0 |
2797 |
GetExpandedInteger(N->getOperand(0), InL, InH); |
0 |
| 2798 |
|
--- |
2798 |
|
--- |
| 2799 |
// If we know that any of the high bits of the shift amount are one, then we |
--- |
2799 |
// If we know that any of the high bits of the shift amount are one, then we |
--- |
| 2800 |
// can do this as a couple of simple shifts. |
--- |
2800 |
// can do this as a couple of simple shifts. |
--- |
| 2801 |
if (Known.One.intersects(HighBitMask)) { |
0 |
2801 |
if (Known.One.intersects(HighBitMask)) { |
0 |
| 2802 |
// Mask out the high bit, which we know is set. |
--- |
2802 |
// Mask out the high bit, which we know is set. |
--- |
| 2803 |
Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt, |
0 |
2803 |
Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt, |
0 |
| 2804 |
DAG.getConstant(~HighBitMask, dl, ShTy)); |
0 |
2804 |
DAG.getConstant(~HighBitMask, dl, ShTy)); |
0 |
| 2805 |
|
--- |
2805 |
|
--- |
| 2806 |
switch (N->getOpcode()) { |
0 |
2806 |
switch (N->getOpcode()) { |
0 |
| 2807 |
default: llvm_unreachable("Unknown shift"); |
0 |
2807 |
default: llvm_unreachable("Unknown shift"); |
0 |
| 2808 |
case ISD::SHL: |
0 |
2808 |
case ISD::SHL: |
0 |
| 2809 |
Lo = DAG.getConstant(0, dl, NVT); // Low part is zero. |
0 |
2809 |
Lo = DAG.getConstant(0, dl, NVT); // Low part is zero. |
0 |
| 2810 |
Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part. |
0 |
2810 |
Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part. |
0 |
| 2811 |
return true; |
0 |
2811 |
return true; |
0 |
| 2812 |
case ISD::SRL: |
0 |
2812 |
case ISD::SRL: |
0 |
| 2813 |
Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero. |
0 |
2813 |
Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero. |
0 |
| 2814 |
Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part. |
0 |
2814 |
Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part. |
0 |
| 2815 |
return true; |
0 |
2815 |
return true; |
0 |
| 2816 |
case ISD::SRA: |
0 |
2816 |
case ISD::SRA: |
0 |
| 2817 |
Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part. |
0 |
2817 |
Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part. |
0 |
| 2818 |
DAG.getConstant(NVTBits - 1, dl, ShTy)); |
0 |
2818 |
DAG.getConstant(NVTBits - 1, dl, ShTy)); |
0 |
| 2819 |
Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part. |
0 |
2819 |
Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part. |
0 |
| 2820 |
return true; |
0 |
2820 |
return true; |
0 |
| 2821 |
} |
--- |
2821 |
} |
--- |
| 2822 |
} |
--- |
2822 |
} |
--- |
| 2823 |
|
--- |
2823 |
|
--- |
| 2824 |
// If we know that all of the high bits of the shift amount are zero, then we |
--- |
2824 |
// If we know that all of the high bits of the shift amount are zero, then we |
--- |
| 2825 |
// can do this as a couple of simple shifts. |
--- |
2825 |
// can do this as a couple of simple shifts. |
--- |
| 2826 |
if (HighBitMask.isSubsetOf(Known.Zero)) { |
0 |
2826 |
if (HighBitMask.isSubsetOf(Known.Zero)) { |
0 |
| 2827 |
// Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined |
--- |
2827 |
// Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined |
--- |
| 2828 |
// shift if x is zero. We can use XOR here because x is known to be smaller |
--- |
2828 |
// shift if x is zero. We can use XOR here because x is known to be smaller |
--- |
| 2829 |
// than 32. |
--- |
2829 |
// than 32. |
--- |
| 2830 |
SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt, |
0 |
2830 |
SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt, |
0 |
| 2831 |
DAG.getConstant(NVTBits - 1, dl, ShTy)); |
0 |
2831 |
DAG.getConstant(NVTBits - 1, dl, ShTy)); |
0 |
| 2832 |
|
--- |
2832 |
|
--- |
| 2833 |
unsigned Op1, Op2; |
--- |
2833 |
unsigned Op1, Op2; |
--- |
| 2834 |
switch (N->getOpcode()) { |
0 |
2834 |
switch (N->getOpcode()) { |
0 |
| 2835 |
default: llvm_unreachable("Unknown shift"); |
0 |
2835 |
default: llvm_unreachable("Unknown shift"); |
0 |
| 2836 |
case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break; |
0 |
2836 |
case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break; |
0 |
| 2837 |
case ISD::SRL: |
0 |
2837 |
case ISD::SRL: |
0 |
| 2838 |
case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break; |
0 |
2838 |
case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break; |
0 |
| 2839 |
} |
--- |
2839 |
} |
--- |
| 2840 |
|
--- |
2840 |
|
--- |
| 2841 |
// When shifting right the arithmetic for Lo and Hi is swapped. |
--- |
2841 |
// When shifting right the arithmetic for Lo and Hi is swapped. |
--- |
| 2842 |
if (N->getOpcode() != ISD::SHL) |
0 |
2842 |
if (N->getOpcode() != ISD::SHL) |
0 |
| 2843 |
std::swap(InL, InH); |
0 |
2843 |
std::swap(InL, InH); |
0 |
| 2844 |
|
--- |
2844 |
|
--- |
| 2845 |
// Use a little trick to get the bits that move from Lo to Hi. First |
--- |
2845 |
// Use a little trick to get the bits that move from Lo to Hi. First |
--- |
| 2846 |
// shift by one bit. |
--- |
2846 |
// shift by one bit. |
--- |
| 2847 |
SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy)); |
0 |
2847 |
SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy)); |
0 |
| 2848 |
// Then compute the remaining shift with amount-1. |
--- |
2848 |
// Then compute the remaining shift with amount-1. |
--- |
| 2849 |
SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2); |
0 |
2849 |
SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2); |
0 |
| 2850 |
|
--- |
2850 |
|
--- |
| 2851 |
Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt); |
0 |
2851 |
Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt); |
0 |
| 2852 |
Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2); |
0 |
2852 |
Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2); |
0 |
| 2853 |
|
--- |
2853 |
|
--- |
| 2854 |
if (N->getOpcode() != ISD::SHL) |
0 |
2854 |
if (N->getOpcode() != ISD::SHL) |
0 |
| 2855 |
std::swap(Hi, Lo); |
0 |
2855 |
std::swap(Hi, Lo); |
0 |
| 2856 |
return true; |
0 |
2856 |
return true; |
0 |
| 2857 |
} |
--- |
2857 |
} |
--- |
| 2858 |
|
--- |
2858 |
|
--- |
| 2859 |
return false; |
0 |
2859 |
return false; |
0 |
| 2860 |
} |
0 |
2860 |
} |
0 |
| 2861 |
|
--- |
2861 |
|
--- |
| 2862 |
/// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift |
--- |
2862 |
/// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift |
--- |
| 2863 |
/// of any size. |
--- |
2863 |
/// of any size. |
--- |
| 2864 |
bool DAGTypeLegalizer:: |
0 |
2864 |
bool DAGTypeLegalizer:: |
0 |
| 2865 |
ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { |
--- |
2865 |
ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { |
--- |
| 2866 |
SDValue Amt = N->getOperand(1); |
0 |
2866 |
SDValue Amt = N->getOperand(1); |
0 |
| 2867 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
2867 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
| 2868 |
EVT ShTy = Amt.getValueType(); |
0 |
2868 |
EVT ShTy = Amt.getValueType(); |
0 |
| 2869 |
unsigned NVTBits = NVT.getSizeInBits(); |
0 |
2869 |
unsigned NVTBits = NVT.getSizeInBits(); |
0 |
| 2870 |
assert(isPowerOf2_32(NVTBits) && |
0 |
2870 |
assert(isPowerOf2_32(NVTBits) && |
0 |
| 2871 |
"Expanded integer type size not a power of two!"); |
--- |
2871 |
"Expanded integer type size not a power of two!"); |
--- |
| 2872 |
SDLoc dl(N); |
0 |
2872 |
SDLoc dl(N); |
0 |
| 2873 |
|
--- |
2873 |
|
--- |
| 2874 |
// Get the incoming operand to be shifted. |
--- |
2874 |
// Get the incoming operand to be shifted. |
--- |
| 2875 |
SDValue InL, InH; |
0 |
2875 |
SDValue InL, InH; |
0 |
| 2876 |
GetExpandedInteger(N->getOperand(0), InL, InH); |
0 |
2876 |
GetExpandedInteger(N->getOperand(0), InL, InH); |
0 |
| 2877 |
|
--- |
2877 |
|
--- |
| 2878 |
SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy); |
0 |
2878 |
SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy); |
0 |
| 2879 |
SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode); |
0 |
2879 |
SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode); |
0 |
| 2880 |
SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt); |
0 |
2880 |
SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt); |
0 |
| 2881 |
SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy), |
0 |
2881 |
SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy), |
0 |
| 2882 |
Amt, NVBitsNode, ISD::SETULT); |
--- |
2882 |
Amt, NVBitsNode, ISD::SETULT); |
--- |
| 2883 |
SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy), |
0 |
2883 |
SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy), |
0 |
| 2884 |
Amt, DAG.getConstant(0, dl, ShTy), |
0 |
2884 |
Amt, DAG.getConstant(0, dl, ShTy), |
0 |
| 2885 |
ISD::SETEQ); |
--- |
2885 |
ISD::SETEQ); |
--- |
| 2886 |
|
--- |
2886 |
|
--- |
| 2887 |
SDValue LoS, HiS, LoL, HiL; |
0 |
2887 |
SDValue LoS, HiS, LoL, HiL; |
0 |
| 2888 |
switch (N->getOpcode()) { |
0 |
2888 |
switch (N->getOpcode()) { |
0 |
| 2889 |
default: llvm_unreachable("Unknown shift"); |
0 |
2889 |
default: llvm_unreachable("Unknown shift"); |
0 |
| 2890 |
case ISD::SHL: |
0 |
2890 |
case ISD::SHL: |
0 |
| 2891 |
// Short: ShAmt < NVTBits |
--- |
2891 |
// Short: ShAmt < NVTBits |
--- |
| 2892 |
LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); |
0 |
2892 |
LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); |
0 |
| 2893 |
HiS = DAG.getNode(ISD::OR, dl, NVT, |
0 |
2893 |
HiS = DAG.getNode(ISD::OR, dl, NVT, |
0 |
| 2894 |
DAG.getNode(ISD::SHL, dl, NVT, InH, Amt), |
0 |
2894 |
DAG.getNode(ISD::SHL, dl, NVT, InH, Amt), |
0 |
| 2895 |
DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack)); |
0 |
2895 |
DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack)); |
0 |
| 2896 |
|
--- |
2896 |
|
--- |
| 2897 |
// Long: ShAmt >= NVTBits |
--- |
2897 |
// Long: ShAmt >= NVTBits |
--- |
| 2898 |
LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero. |
0 |
2898 |
LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero. |
0 |
| 2899 |
HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part. |
0 |
2899 |
HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part. |
0 |
| 2900 |
|
--- |
2900 |
|
--- |
| 2901 |
Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL); |
0 |
2901 |
Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL); |
0 |
| 2902 |
Hi = DAG.getSelect(dl, NVT, isZero, InH, |
0 |
2902 |
Hi = DAG.getSelect(dl, NVT, isZero, InH, |
0 |
| 2903 |
DAG.getSelect(dl, NVT, isShort, HiS, HiL)); |
0 |
2903 |
DAG.getSelect(dl, NVT, isShort, HiS, HiL)); |
0 |
| 2904 |
return true; |
0 |
2904 |
return true; |
0 |
| 2905 |
case ISD::SRL: |
0 |
2905 |
case ISD::SRL: |
0 |
| 2906 |
// Short: ShAmt < NVTBits |
--- |
2906 |
// Short: ShAmt < NVTBits |
--- |
| 2907 |
HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); |
0 |
2907 |
HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); |
0 |
| 2908 |
LoS = DAG.getNode(ISD::OR, dl, NVT, |
0 |
2908 |
LoS = DAG.getNode(ISD::OR, dl, NVT, |
0 |
| 2909 |
DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), |
0 |
2909 |
DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), |
0 |
| 2910 |
// FIXME: If Amt is zero, the following shift generates an undefined result |
--- |
2910 |
// FIXME: If Amt is zero, the following shift generates an undefined result |
--- |
| 2911 |
// on some architectures. |
--- |
2911 |
// on some architectures. |
--- |
| 2912 |
DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); |
0 |
2912 |
DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); |
0 |
| 2913 |
|
--- |
2913 |
|
--- |
| 2914 |
// Long: ShAmt >= NVTBits |
--- |
2914 |
// Long: ShAmt >= NVTBits |
--- |
| 2915 |
HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero. |
0 |
2915 |
HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero. |
0 |
| 2916 |
LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part. |
0 |
2916 |
LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part. |
0 |
| 2917 |
|
--- |
2917 |
|
--- |
| 2918 |
Lo = DAG.getSelect(dl, NVT, isZero, InL, |
0 |
2918 |
Lo = DAG.getSelect(dl, NVT, isZero, InL, |
0 |
| 2919 |
DAG.getSelect(dl, NVT, isShort, LoS, LoL)); |
0 |
2919 |
DAG.getSelect(dl, NVT, isShort, LoS, LoL)); |
0 |
| 2920 |
Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL); |
0 |
2920 |
Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL); |
0 |
| 2921 |
return true; |
0 |
2921 |
return true; |
0 |
| 2922 |
case ISD::SRA: |
0 |
2922 |
case ISD::SRA: |
0 |
| 2923 |
// Short: ShAmt < NVTBits |
--- |
2923 |
// Short: ShAmt < NVTBits |
--- |
| 2924 |
HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); |
0 |
2924 |
HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); |
0 |
| 2925 |
LoS = DAG.getNode(ISD::OR, dl, NVT, |
0 |
2925 |
LoS = DAG.getNode(ISD::OR, dl, NVT, |
0 |
| 2926 |
DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), |
0 |
2926 |
DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), |
0 |
| 2927 |
DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); |
0 |
2927 |
DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); |
0 |
| 2928 |
|
--- |
2928 |
|
--- |
| 2929 |
// Long: ShAmt >= NVTBits |
--- |
2929 |
// Long: ShAmt >= NVTBits |
--- |
| 2930 |
HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part. |
0 |
2930 |
HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part. |
0 |
| 2931 |
DAG.getConstant(NVTBits - 1, dl, ShTy)); |
0 |
2931 |
DAG.getConstant(NVTBits - 1, dl, ShTy)); |
0 |
| 2932 |
LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part. |
0 |
2932 |
LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part. |
0 |
| 2933 |
|
--- |
2933 |
|
--- |
| 2934 |
Lo = DAG.getSelect(dl, NVT, isZero, InL, |
0 |
2934 |
Lo = DAG.getSelect(dl, NVT, isZero, InL, |
0 |
| 2935 |
DAG.getSelect(dl, NVT, isShort, LoS, LoL)); |
0 |
2935 |
DAG.getSelect(dl, NVT, isShort, LoS, LoL)); |
0 |
| 2936 |
Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL); |
0 |
2936 |
Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL); |
0 |
| 2937 |
return true; |
0 |
2937 |
return true; |
0 |
| 2938 |
} |
--- |
2938 |
} |
--- |
| 2939 |
} |
0 |
2939 |
} |
0 |
| 2940 |
|
--- |
2940 |
|
--- |
| 2941 |
static std::pair getExpandedMinMaxOps(int Op) { |
0 |
2941 |
static std::pair getExpandedMinMaxOps(int Op) { |
0 |
| 2942 |
|
--- |
2942 |
|
--- |
| 2943 |
switch (Op) { |
0 |
2943 |
switch (Op) { |
0 |
| 2944 |
default: llvm_unreachable("invalid min/max opcode"); |
0 |
2944 |
default: llvm_unreachable("invalid min/max opcode"); |
0 |
| 2945 |
case ISD::SMAX: |
0 |
2945 |
case ISD::SMAX: |
0 |
| 2946 |
return std::make_pair(ISD::SETGT, ISD::UMAX); |
0 |
2946 |
return std::make_pair(ISD::SETGT, ISD::UMAX); |
0 |
| 2947 |
case ISD::UMAX: |
0 |
2947 |
case ISD::UMAX: |
0 |
| 2948 |
return std::make_pair(ISD::SETUGT, ISD::UMAX); |
0 |
2948 |
return std::make_pair(ISD::SETUGT, ISD::UMAX); |
0 |
| 2949 |
case ISD::SMIN: |
0 |
2949 |
case ISD::SMIN: |
0 |
| 2950 |
return std::make_pair(ISD::SETLT, ISD::UMIN); |
0 |
2950 |
return std::make_pair(ISD::SETLT, ISD::UMIN); |
0 |
| 2951 |
case ISD::UMIN: |
0 |
2951 |
case ISD::UMIN: |
0 |
| 2952 |
return std::make_pair(ISD::SETULT, ISD::UMIN); |
0 |
2952 |
return std::make_pair(ISD::SETULT, ISD::UMIN); |
0 |
| 2953 |
} |
--- |
2953 |
} |
--- |
| 2954 |
} |
--- |
2954 |
} |
--- |
| 2955 |
|
--- |
2955 |
|
--- |
| 2956 |
void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N, |
0 |
2956 |
void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N, |
0 |
| 2957 |
SDValue &Lo, SDValue &Hi) { |
--- |
2957 |
SDValue &Lo, SDValue &Hi) { |
--- |
| 2958 |
SDLoc DL(N); |
0 |
2958 |
SDLoc DL(N); |
0 |
| 2959 |
|
--- |
2959 |
|
--- |
| 2960 |
SDValue LHS = N->getOperand(0); |
0 |
2960 |
SDValue LHS = N->getOperand(0); |
0 |
| 2961 |
SDValue RHS = N->getOperand(1); |
0 |
2961 |
SDValue RHS = N->getOperand(1); |
0 |
| 2962 |
|
--- |
2962 |
|
--- |
| 2963 |
// If the upper halves are all sign bits, then we can perform the MINMAX on |
--- |
2963 |
// If the upper halves are all sign bits, then we can perform the MINMAX on |
--- |
| 2964 |
// the lower half and sign-extend the result to the upper half. |
--- |
2964 |
// the lower half and sign-extend the result to the upper half. |
--- |
| 2965 |
unsigned NumBits = N->getValueType(0).getScalarSizeInBits(); |
0 |
2965 |
unsigned NumBits = N->getValueType(0).getScalarSizeInBits(); |
0 |
| 2966 |
unsigned NumHalfBits = NumBits / 2; |
0 |
2966 |
unsigned NumHalfBits = NumBits / 2; |
0 |
| 2967 |
if (DAG.ComputeNumSignBits(LHS) > NumHalfBits && |
0 |
2967 |
if (DAG.ComputeNumSignBits(LHS) > NumHalfBits && |
0 |
| 2968 |
DAG.ComputeNumSignBits(RHS) > NumHalfBits) { |
0 |
2968 |
DAG.ComputeNumSignBits(RHS) > NumHalfBits) { |
0 |
| 2969 |
SDValue LHSL, LHSH, RHSL, RHSH; |
0 |
2969 |
SDValue LHSL, LHSH, RHSL, RHSH; |
0 |
| 2970 |
GetExpandedInteger(LHS, LHSL, LHSH); |
0 |
2970 |
GetExpandedInteger(LHS, LHSL, LHSH); |
0 |
| 2971 |
GetExpandedInteger(RHS, RHSL, RHSH); |
0 |
2971 |
GetExpandedInteger(RHS, RHSL, RHSH); |
0 |
| 2972 |
EVT NVT = LHSL.getValueType(); |
0 |
2972 |
EVT NVT = LHSL.getValueType(); |
0 |
| 2973 |
|
--- |
2973 |
|
--- |
| 2974 |
Lo = DAG.getNode(N->getOpcode(), DL, NVT, LHSL, RHSL); |
0 |
2974 |
Lo = DAG.getNode(N->getOpcode(), DL, NVT, LHSL, RHSL); |
0 |
| 2975 |
Hi = DAG.getNode(ISD::SRA, DL, NVT, Lo, |
0 |
2975 |
Hi = DAG.getNode(ISD::SRA, DL, NVT, Lo, |
0 |
| 2976 |
DAG.getShiftAmountConstant(NumHalfBits - 1, NVT, DL)); |
0 |
2976 |
DAG.getShiftAmountConstant(NumHalfBits - 1, NVT, DL)); |
0 |
| 2977 |
return; |
0 |
2977 |
return; |
0 |
| 2978 |
} |
--- |
2978 |
} |
--- |
| 2979 |
|
--- |
2979 |
|
--- |
| 2980 |
// The Lo of smin(X, -1) is LHSL if X is negative. Otherwise it's -1. |
--- |
2980 |
// The Lo of smin(X, -1) is LHSL if X is negative. Otherwise it's -1. |
--- |
| 2981 |
// The Lo of smax(X, 0) is 0 if X is negative. Otherwise it's LHSL. |
--- |
2981 |
// The Lo of smax(X, 0) is 0 if X is negative. Otherwise it's LHSL. |
--- |
| 2982 |
if ((N->getOpcode() == ISD::SMAX && isNullConstant(RHS)) || |
0 |
2982 |
if ((N->getOpcode() == ISD::SMAX && isNullConstant(RHS)) || |
0 |
| 2983 |
(N->getOpcode() == ISD::SMIN && isAllOnesConstant(RHS))) { |
0 |
2983 |
(N->getOpcode() == ISD::SMIN && isAllOnesConstant(RHS))) { |
0 |
| 2984 |
SDValue LHSL, LHSH, RHSL, RHSH; |
0 |
2984 |
SDValue LHSL, LHSH, RHSL, RHSH; |
0 |
| 2985 |
GetExpandedInteger(LHS, LHSL, LHSH); |
0 |
2985 |
GetExpandedInteger(LHS, LHSL, LHSH); |
0 |
| 2986 |
GetExpandedInteger(RHS, RHSL, RHSH); |
0 |
2986 |
GetExpandedInteger(RHS, RHSL, RHSH); |
0 |
| 2987 |
EVT NVT = LHSL.getValueType(); |
0 |
2987 |
EVT NVT = LHSL.getValueType(); |
0 |
| 2988 |
EVT CCT = getSetCCResultType(NVT); |
0 |
2988 |
EVT CCT = getSetCCResultType(NVT); |
0 |
| 2989 |
|
--- |
2989 |
|
--- |
| 2990 |
SDValue HiNeg = |
--- |
2990 |
SDValue HiNeg = |
--- |
| 2991 |
DAG.getSetCC(DL, CCT, LHSH, DAG.getConstant(0, DL, NVT), ISD::SETLT); |
0 |
2991 |
DAG.getSetCC(DL, CCT, LHSH, DAG.getConstant(0, DL, NVT), ISD::SETLT); |
0 |
| 2992 |
if (N->getOpcode() == ISD::SMIN) { |
0 |
2992 |
if (N->getOpcode() == ISD::SMIN) { |
0 |
| 2993 |
Lo = DAG.getSelect(DL, NVT, HiNeg, LHSL, DAG.getConstant(-1, DL, NVT)); |
0 |
2993 |
Lo = DAG.getSelect(DL, NVT, HiNeg, LHSL, DAG.getConstant(-1, DL, NVT)); |
0 |
| 2994 |
} else { |
--- |
2994 |
} else { |
--- |
| 2995 |
Lo = DAG.getSelect(DL, NVT, HiNeg, DAG.getConstant(0, DL, NVT), LHSL); |
0 |
2995 |
Lo = DAG.getSelect(DL, NVT, HiNeg, DAG.getConstant(0, DL, NVT), LHSL); |
0 |
| 2996 |
} |
--- |
2996 |
} |
--- |
| 2997 |
Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH}); |
0 |
2997 |
Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH}); |
0 |
| 2998 |
return; |
0 |
2998 |
return; |
0 |
| 2999 |
} |
--- |
2999 |
} |
--- |
| 3000 |
|
--- |
3000 |
|
--- |
| 3001 |
const APInt *RHSVal = nullptr; |
0 |
3001 |
const APInt *RHSVal = nullptr; |
0 |
| 3002 |
if (auto *RHSConst = dyn_cast(RHS)) |
0 |
3002 |
if (auto *RHSConst = dyn_cast(RHS)) |
0 |
| 3003 |
RHSVal = &RHSConst->getAPIntValue(); |
0 |
3003 |
RHSVal = &RHSConst->getAPIntValue(); |
0 |
| 3004 |
|
--- |
3004 |
|
--- |
| 3005 |
// The high half of MIN/MAX is always just the the MIN/MAX of the |
--- |
3005 |
// The high half of MIN/MAX is always just the the MIN/MAX of the |
--- |
| 3006 |
// high halves of the operands. Expand this way if it appears profitable. |
--- |
3006 |
// high halves of the operands. Expand this way if it appears profitable. |
--- |
| 3007 |
if (RHSVal && (N->getOpcode() == ISD::UMIN || N->getOpcode() == ISD::UMAX) && |
0 |
3007 |
if (RHSVal && (N->getOpcode() == ISD::UMIN || N->getOpcode() == ISD::UMAX) && |
0 |
| 3008 |
(RHSVal->countLeadingOnes() >= NumHalfBits || |
0 |
3008 |
(RHSVal->countLeadingOnes() >= NumHalfBits || |
0 |
| 3009 |
RHSVal->countLeadingZeros() >= NumHalfBits)) { |
0 |
3009 |
RHSVal->countLeadingZeros() >= NumHalfBits)) { |
0 |
| 3010 |
SDValue LHSL, LHSH, RHSL, RHSH; |
0 |
3010 |
SDValue LHSL, LHSH, RHSL, RHSH; |
0 |
| 3011 |
GetExpandedInteger(LHS, LHSL, LHSH); |
0 |
3011 |
GetExpandedInteger(LHS, LHSL, LHSH); |
0 |
| 3012 |
GetExpandedInteger(RHS, RHSL, RHSH); |
0 |
3012 |
GetExpandedInteger(RHS, RHSL, RHSH); |
0 |
| 3013 |
EVT NVT = LHSL.getValueType(); |
0 |
3013 |
EVT NVT = LHSL.getValueType(); |
0 |
| 3014 |
EVT CCT = getSetCCResultType(NVT); |
0 |
3014 |
EVT CCT = getSetCCResultType(NVT); |
0 |
| 3015 |
|
--- |
3015 |
|
--- |
| 3016 |
ISD::NodeType LoOpc; |
--- |
3016 |
ISD::NodeType LoOpc; |
--- |
| 3017 |
ISD::CondCode CondC; |
--- |
3017 |
ISD::CondCode CondC; |
--- |
| 3018 |
std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode()); |
0 |
3018 |
std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode()); |
0 |
| 3019 |
|
--- |
3019 |
|
--- |
| 3020 |
Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH}); |
0 |
3020 |
Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH}); |
0 |
| 3021 |
// We need to know whether to select Lo part that corresponds to 'winning' |
--- |
3021 |
// We need to know whether to select Lo part that corresponds to 'winning' |
--- |
| 3022 |
// Hi part or if Hi parts are equal. |
--- |
3022 |
// Hi part or if Hi parts are equal. |
--- |
| 3023 |
SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC); |
0 |
3023 |
SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC); |
0 |
| 3024 |
SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ); |
0 |
3024 |
SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ); |
0 |
| 3025 |
|
--- |
3025 |
|
--- |
| 3026 |
// Lo part corresponding to the 'winning' Hi part |
--- |
3026 |
// Lo part corresponding to the 'winning' Hi part |
--- |
| 3027 |
SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL); |
0 |
3027 |
SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL); |
0 |
| 3028 |
|
--- |
3028 |
|
--- |
| 3029 |
// Recursed Lo part if Hi parts are equal, this uses unsigned version |
--- |
3029 |
// Recursed Lo part if Hi parts are equal, this uses unsigned version |
--- |
| 3030 |
SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL}); |
0 |
3030 |
SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL}); |
0 |
| 3031 |
|
--- |
3031 |
|
--- |
| 3032 |
Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp); |
0 |
3032 |
Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp); |
0 |
| 3033 |
return; |
0 |
3033 |
return; |
0 |
| 3034 |
} |
--- |
3034 |
} |
--- |
| 3035 |
|
--- |
3035 |
|
--- |
| 3036 |
// Expand to "a < b ? a : b" etc. Prefer ge/le if that simplifies |
--- |
3036 |
// Expand to "a < b ? a : b" etc. Prefer ge/le if that simplifies |
--- |
| 3037 |
// the compare. |
--- |
3037 |
// the compare. |
--- |
| 3038 |
ISD::CondCode Pred; |
--- |
3038 |
ISD::CondCode Pred; |
--- |
| 3039 |
switch (N->getOpcode()) { |
0 |
3039 |
switch (N->getOpcode()) { |
0 |
| 3040 |
default: llvm_unreachable("How did we get here?"); |
0 |
3040 |
default: llvm_unreachable("How did we get here?"); |
0 |
| 3041 |
case ISD::SMAX: |
0 |
3041 |
case ISD::SMAX: |
0 |
| 3042 |
if (RHSVal && RHSVal->countTrailingZeros() >= NumHalfBits) |
0 |
3042 |
if (RHSVal && RHSVal->countTrailingZeros() >= NumHalfBits) |
0 |
| 3043 |
Pred = ISD::SETGE; |
0 |
3043 |
Pred = ISD::SETGE; |
0 |
| 3044 |
else |
--- |
3044 |
else |
--- |
| 3045 |
Pred = ISD::SETGT; |
0 |
3045 |
Pred = ISD::SETGT; |
0 |
| 3046 |
break; |
0 |
3046 |
break; |
0 |
| 3047 |
case ISD::SMIN: |
0 |
3047 |
case ISD::SMIN: |
0 |
| 3048 |
if (RHSVal && RHSVal->countTrailingOnes() >= NumHalfBits) |
0 |
3048 |
if (RHSVal && RHSVal->countTrailingOnes() >= NumHalfBits) |
0 |
| 3049 |
Pred = ISD::SETLE; |
0 |
3049 |
Pred = ISD::SETLE; |
0 |
| 3050 |
else |
--- |
3050 |
else |
--- |
| 3051 |
Pred = ISD::SETLT; |
0 |
3051 |
Pred = ISD::SETLT; |
0 |
| 3052 |
break; |
0 |
3052 |
break; |
0 |
| 3053 |
case ISD::UMAX: |
0 |
3053 |
case ISD::UMAX: |
0 |
| 3054 |
if (RHSVal && RHSVal->countTrailingZeros() >= NumHalfBits) |
0 |
3054 |
if (RHSVal && RHSVal->countTrailingZeros() >= NumHalfBits) |
0 |
| 3055 |
Pred = ISD::SETUGE; |
0 |
3055 |
Pred = ISD::SETUGE; |
0 |
| 3056 |
else |
--- |
3056 |
else |
--- |
| 3057 |
Pred = ISD::SETUGT; |
0 |
3057 |
Pred = ISD::SETUGT; |
0 |
| 3058 |
break; |
0 |
3058 |
break; |
0 |
| 3059 |
case ISD::UMIN: |
0 |
3059 |
case ISD::UMIN: |
0 |
| 3060 |
if (RHSVal && RHSVal->countTrailingOnes() >= NumHalfBits) |
0 |
3060 |
if (RHSVal && RHSVal->countTrailingOnes() >= NumHalfBits) |
0 |
| 3061 |
Pred = ISD::SETULE; |
0 |
3061 |
Pred = ISD::SETULE; |
0 |
| 3062 |
else |
--- |
3062 |
else |
--- |
| 3063 |
Pred = ISD::SETULT; |
0 |
3063 |
Pred = ISD::SETULT; |
0 |
| 3064 |
break; |
0 |
3064 |
break; |
0 |
| 3065 |
} |
--- |
3065 |
} |
--- |
| 3066 |
EVT VT = N->getValueType(0); |
0 |
3066 |
EVT VT = N->getValueType(0); |
0 |
| 3067 |
EVT CCT = getSetCCResultType(VT); |
0 |
3067 |
EVT CCT = getSetCCResultType(VT); |
0 |
| 3068 |
SDValue Cond = DAG.getSetCC(DL, CCT, LHS, RHS, Pred); |
0 |
3068 |
SDValue Cond = DAG.getSetCC(DL, CCT, LHS, RHS, Pred); |
0 |
| 3069 |
SDValue Result = DAG.getSelect(DL, VT, Cond, LHS, RHS); |
0 |
3069 |
SDValue Result = DAG.getSelect(DL, VT, Cond, LHS, RHS); |
0 |
| 3070 |
SplitInteger(Result, Lo, Hi); |
0 |
3070 |
SplitInteger(Result, Lo, Hi); |
0 |
| 3071 |
} |
0 |
3071 |
} |
0 |
| 3072 |
|
--- |
3072 |
|
--- |
| 3073 |
void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N, |
0 |
3073 |
void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N, |
0 |
| 3074 |
SDValue &Lo, SDValue &Hi) { |
--- |
3074 |
SDValue &Lo, SDValue &Hi) { |
--- |
| 3075 |
SDLoc dl(N); |
0 |
3075 |
SDLoc dl(N); |
0 |
| 3076 |
// Expand the subcomponents. |
--- |
3076 |
// Expand the subcomponents. |
--- |
| 3077 |
SDValue LHSL, LHSH, RHSL, RHSH; |
0 |
3077 |
SDValue LHSL, LHSH, RHSL, RHSH; |
0 |
| 3078 |
GetExpandedInteger(N->getOperand(0), LHSL, LHSH); |
0 |
3078 |
GetExpandedInteger(N->getOperand(0), LHSL, LHSH); |
0 |
| 3079 |
GetExpandedInteger(N->getOperand(1), RHSL, RHSH); |
0 |
3079 |
GetExpandedInteger(N->getOperand(1), RHSL, RHSH); |
0 |
| 3080 |
|
--- |
3080 |
|
--- |
| 3081 |
EVT NVT = LHSL.getValueType(); |
0 |
3081 |
EVT NVT = LHSL.getValueType(); |
0 |
| 3082 |
SDValue LoOps[2] = { LHSL, RHSL }; |
0 |
3082 |
SDValue LoOps[2] = { LHSL, RHSL }; |
0 |
| 3083 |
SDValue HiOps[3] = { LHSH, RHSH }; |
0 |
3083 |
SDValue HiOps[3] = { LHSH, RHSH }; |
0 |
| 3084 |
|
--- |
3084 |
|
--- |
| 3085 |
bool HasOpCarry = TLI.isOperationLegalOrCustom( |
0 |
3085 |
bool HasOpCarry = TLI.isOperationLegalOrCustom( |
0 |
| 3086 |
N->getOpcode() == ISD::ADD ? ISD::UADDO_CARRY : ISD::USUBO_CARRY, |
0 |
3086 |
N->getOpcode() == ISD::ADD ? ISD::UADDO_CARRY : ISD::USUBO_CARRY, |
0 |
| 3087 |
TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); |
0 |
3087 |
TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); |
0 |
| 3088 |
if (HasOpCarry) { |
0 |
3088 |
if (HasOpCarry) { |
0 |
| 3089 |
SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT)); |
0 |
3089 |
SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT)); |
0 |
| 3090 |
if (N->getOpcode() == ISD::ADD) { |
0 |
3090 |
if (N->getOpcode() == ISD::ADD) { |
0 |
| 3091 |
Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps); |
0 |
3091 |
Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps); |
0 |
| 3092 |
HiOps[2] = Lo.getValue(1); |
0 |
3092 |
HiOps[2] = Lo.getValue(1); |
0 |
| 3093 |
Hi = DAG.computeKnownBits(HiOps[2]).isZero() |
0 |
3093 |
Hi = DAG.computeKnownBits(HiOps[2]).isZero() |
0 |
| 3094 |
? DAG.getNode(ISD::UADDO, dl, VTList, ArrayRef(HiOps, 2)) |
0 |
3094 |
? DAG.getNode(ISD::UADDO, dl, VTList, ArrayRef(HiOps, 2)) |
0 |
| 3095 |
: DAG.getNode(ISD::UADDO_CARRY, dl, VTList, HiOps); |
0 |
3095 |
: DAG.getNode(ISD::UADDO_CARRY, dl, VTList, HiOps); |
0 |
| 3096 |
} else { |
--- |
3096 |
} else { |
--- |
| 3097 |
Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps); |
0 |
3097 |
Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps); |
0 |
| 3098 |
HiOps[2] = Lo.getValue(1); |
0 |
3098 |
HiOps[2] = Lo.getValue(1); |
0 |
| 3099 |
Hi = DAG.computeKnownBits(HiOps[2]).isZero() |
0 |
3099 |
Hi = DAG.computeKnownBits(HiOps[2]).isZero() |
0 |
| 3100 |
? DAG.getNode(ISD::USUBO, dl, VTList, ArrayRef(HiOps, 2)) |
0 |
3100 |
? DAG.getNode(ISD::USUBO, dl, VTList, ArrayRef(HiOps, 2)) |
0 |
| 3101 |
: DAG.getNode(ISD::USUBO_CARRY, dl, VTList, HiOps); |
0 |
3101 |
: DAG.getNode(ISD::USUBO_CARRY, dl, VTList, HiOps); |
0 |
| 3102 |
} |
--- |
3102 |
} |
--- |
| 3103 |
return; |
0 |
3103 |
return; |
0 |
| 3104 |
} |
--- |
3104 |
} |
--- |
| 3105 |
|
--- |
3105 |
|
--- |
| 3106 |
// Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support |
--- |
3106 |
// Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support |
--- |
| 3107 |
// them. TODO: Teach operation legalization how to expand unsupported |
--- |
3107 |
// them. TODO: Teach operation legalization how to expand unsupported |
--- |
| 3108 |
// ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate |
--- |
3108 |
// ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate |
--- |
| 3109 |
// a carry of type MVT::Glue, but there doesn't seem to be any way to |
--- |
3109 |
// a carry of type MVT::Glue, but there doesn't seem to be any way to |
--- |
| 3110 |
// generate a value of this type in the expanded code sequence. |
--- |
3110 |
// generate a value of this type in the expanded code sequence. |
--- |
| 3111 |
bool hasCarry = |
--- |
3111 |
bool hasCarry = |
--- |
| 3112 |
TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ? |
0 |
3112 |
TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ? |
0 |
| 3113 |
ISD::ADDC : ISD::SUBC, |
--- |
3113 |
ISD::ADDC : ISD::SUBC, |
--- |
| 3114 |
TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); |
0 |
3114 |
TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); |
0 |
| 3115 |
|
--- |
3115 |
|
--- |
| 3116 |
if (hasCarry) { |
0 |
3116 |
if (hasCarry) { |
0 |
| 3117 |
SDVTList VTList = DAG.getVTList(NVT, MVT::Glue); |
0 |
3117 |
SDVTList VTList = DAG.getVTList(NVT, MVT::Glue); |
0 |
| 3118 |
if (N->getOpcode() == ISD::ADD) { |
0 |
3118 |
if (N->getOpcode() == ISD::ADD) { |
0 |
| 3119 |
Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps); |
0 |
3119 |
Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps); |
0 |
| 3120 |
HiOps[2] = Lo.getValue(1); |
0 |
3120 |
HiOps[2] = Lo.getValue(1); |
0 |
| 3121 |
Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps); |
0 |
3121 |
Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps); |
0 |
| 3122 |
} else { |
--- |
3122 |
} else { |
--- |
| 3123 |
Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps); |
0 |
3123 |
Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps); |
0 |
| 3124 |
HiOps[2] = Lo.getValue(1); |
0 |
3124 |
HiOps[2] = Lo.getValue(1); |
0 |
| 3125 |
Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps); |
0 |
3125 |
Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps); |
0 |
| 3126 |
} |
--- |
3126 |
} |
--- |
| 3127 |
return; |
0 |
3127 |
return; |
0 |
| 3128 |
} |
--- |
3128 |
} |
--- |
| 3129 |
|
--- |
3129 |
|
--- |
| 3130 |
bool hasOVF = |
--- |
3130 |
bool hasOVF = |
--- |
| 3131 |
TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ? |
0 |
3131 |
TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ? |
0 |
| 3132 |
ISD::UADDO : ISD::USUBO, |
--- |
3132 |
ISD::UADDO : ISD::USUBO, |
--- |
| 3133 |
TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); |
0 |
3133 |
TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); |
0 |
| 3134 |
TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(NVT); |
0 |
3134 |
TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(NVT); |
0 |
| 3135 |
|
--- |
3135 |
|
--- |
| 3136 |
if (hasOVF) { |
0 |
3136 |
if (hasOVF) { |
0 |
| 3137 |
EVT OvfVT = getSetCCResultType(NVT); |
0 |
3137 |
EVT OvfVT = getSetCCResultType(NVT); |
0 |
| 3138 |
SDVTList VTList = DAG.getVTList(NVT, OvfVT); |
0 |
3138 |
SDVTList VTList = DAG.getVTList(NVT, OvfVT); |
0 |
| 3139 |
int RevOpc; |
--- |
3139 |
int RevOpc; |
--- |
| 3140 |
if (N->getOpcode() == ISD::ADD) { |
0 |
3140 |
if (N->getOpcode() == ISD::ADD) { |
0 |
| 3141 |
RevOpc = ISD::SUB; |
0 |
3141 |
RevOpc = ISD::SUB; |
0 |
| 3142 |
Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps); |
0 |
3142 |
Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps); |
0 |
| 3143 |
Hi = DAG.getNode(ISD::ADD, dl, NVT, ArrayRef(HiOps, 2)); |
0 |
3143 |
Hi = DAG.getNode(ISD::ADD, dl, NVT, ArrayRef(HiOps, 2)); |
0 |
| 3144 |
} else { |
--- |
3144 |
} else { |
--- |
| 3145 |
RevOpc = ISD::ADD; |
0 |
3145 |
RevOpc = ISD::ADD; |
0 |
| 3146 |
Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps); |
0 |
3146 |
Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps); |
0 |
| 3147 |
Hi = DAG.getNode(ISD::SUB, dl, NVT, ArrayRef(HiOps, 2)); |
0 |
3147 |
Hi = DAG.getNode(ISD::SUB, dl, NVT, ArrayRef(HiOps, 2)); |
0 |
| 3148 |
} |
--- |
3148 |
} |
--- |
| 3149 |
SDValue OVF = Lo.getValue(1); |
0 |
3149 |
SDValue OVF = Lo.getValue(1); |
0 |
| 3150 |
|
--- |
3150 |
|
--- |
| 3151 |
switch (BoolType) { |
0 |
3151 |
switch (BoolType) { |
0 |
| 3152 |
case TargetLoweringBase::UndefinedBooleanContent: |
0 |
3152 |
case TargetLoweringBase::UndefinedBooleanContent: |
0 |
| 3153 |
OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF); |
0 |
3153 |
OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF); |
0 |
| 3154 |
[[fallthrough]]; |
--- |
3154 |
[[fallthrough]]; |
--- |
| 3155 |
case TargetLoweringBase::ZeroOrOneBooleanContent: |
0 |
3155 |
case TargetLoweringBase::ZeroOrOneBooleanContent: |
0 |
| 3156 |
OVF = DAG.getZExtOrTrunc(OVF, dl, NVT); |
0 |
3156 |
OVF = DAG.getZExtOrTrunc(OVF, dl, NVT); |
0 |
| 3157 |
Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF); |
0 |
3157 |
Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF); |
0 |
| 3158 |
break; |
0 |
3158 |
break; |
0 |
| 3159 |
case TargetLoweringBase::ZeroOrNegativeOneBooleanContent: |
0 |
3159 |
case TargetLoweringBase::ZeroOrNegativeOneBooleanContent: |
0 |
| 3160 |
OVF = DAG.getSExtOrTrunc(OVF, dl, NVT); |
0 |
3160 |
OVF = DAG.getSExtOrTrunc(OVF, dl, NVT); |
0 |
| 3161 |
Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF); |
0 |
3161 |
Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF); |
0 |
| 3162 |
} |
--- |
3162 |
} |
--- |
| 3163 |
return; |
0 |
3163 |
return; |
0 |
| 3164 |
} |
--- |
3164 |
} |
--- |
| 3165 |
|
--- |
3165 |
|
--- |
| 3166 |
if (N->getOpcode() == ISD::ADD) { |
0 |
3166 |
if (N->getOpcode() == ISD::ADD) { |
0 |
| 3167 |
Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps); |
0 |
3167 |
Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps); |
0 |
| 3168 |
Hi = DAG.getNode(ISD::ADD, dl, NVT, ArrayRef(HiOps, 2)); |
0 |
3168 |
Hi = DAG.getNode(ISD::ADD, dl, NVT, ArrayRef(HiOps, 2)); |
0 |
| 3169 |
SDValue Cmp; |
0 |
3169 |
SDValue Cmp; |
0 |
| 3170 |
// Special case: X+1 has a carry out if X+1==0. This may reduce the live |
--- |
3170 |
// Special case: X+1 has a carry out if X+1==0. This may reduce the live |
--- |
| 3171 |
// range of X. We assume comparing with 0 is cheap. |
--- |
3171 |
// range of X. We assume comparing with 0 is cheap. |
--- |
| 3172 |
if (isOneConstant(LoOps[1])) |
0 |
3172 |
if (isOneConstant(LoOps[1])) |
0 |
| 3173 |
Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, |
0 |
3173 |
Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, |
0 |
| 3174 |
DAG.getConstant(0, dl, NVT), ISD::SETEQ); |
0 |
3174 |
DAG.getConstant(0, dl, NVT), ISD::SETEQ); |
0 |
| 3175 |
else if (isAllOnesConstant(LoOps[1])) { |
0 |
3175 |
else if (isAllOnesConstant(LoOps[1])) { |
0 |
| 3176 |
if (isAllOnesConstant(HiOps[1])) |
0 |
3176 |
if (isAllOnesConstant(HiOps[1])) |
0 |
| 3177 |
Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), LoOps[0], |
0 |
3177 |
Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), LoOps[0], |
0 |
| 3178 |
DAG.getConstant(0, dl, NVT), ISD::SETEQ); |
0 |
3178 |
DAG.getConstant(0, dl, NVT), ISD::SETEQ); |
0 |
| 3179 |
else |
--- |
3179 |
else |
--- |
| 3180 |
Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), LoOps[0], |
0 |
3180 |
Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), LoOps[0], |
0 |
| 3181 |
DAG.getConstant(0, dl, NVT), ISD::SETNE); |
0 |
3181 |
DAG.getConstant(0, dl, NVT), ISD::SETNE); |
0 |
| 3182 |
} else |
--- |
3182 |
} else |
--- |
| 3183 |
Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0], |
0 |
3183 |
Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0], |
0 |
| 3184 |
ISD::SETULT); |
--- |
3184 |
ISD::SETULT); |
--- |
| 3185 |
|
--- |
3185 |
|
--- |
| 3186 |
SDValue Carry; |
0 |
3186 |
SDValue Carry; |
0 |
| 3187 |
if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) |
0 |
3187 |
if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) |
0 |
| 3188 |
Carry = DAG.getZExtOrTrunc(Cmp, dl, NVT); |
0 |
3188 |
Carry = DAG.getZExtOrTrunc(Cmp, dl, NVT); |
0 |
| 3189 |
else |
--- |
3189 |
else |
--- |
| 3190 |
Carry = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT), |
0 |
3190 |
Carry = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT), |
0 |
| 3191 |
DAG.getConstant(0, dl, NVT)); |
0 |
3191 |
DAG.getConstant(0, dl, NVT)); |
0 |
| 3192 |
|
--- |
3192 |
|
--- |
| 3193 |
if (isAllOnesConstant(LoOps[1]) && isAllOnesConstant(HiOps[1])) |
0 |
3193 |
if (isAllOnesConstant(LoOps[1]) && isAllOnesConstant(HiOps[1])) |
0 |
| 3194 |
Hi = DAG.getNode(ISD::SUB, dl, NVT, HiOps[0], Carry); |
0 |
3194 |
Hi = DAG.getNode(ISD::SUB, dl, NVT, HiOps[0], Carry); |
0 |
| 3195 |
else |
--- |
3195 |
else |
--- |
| 3196 |
Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry); |
0 |
3196 |
Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry); |
0 |
| 3197 |
} else { |
--- |
3197 |
} else { |
--- |
| 3198 |
Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps); |
0 |
3198 |
Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps); |
0 |
| 3199 |
Hi = DAG.getNode(ISD::SUB, dl, NVT, ArrayRef(HiOps, 2)); |
0 |
3199 |
Hi = DAG.getNode(ISD::SUB, dl, NVT, ArrayRef(HiOps, 2)); |
0 |
| 3200 |
SDValue Cmp = |
--- |
3200 |
SDValue Cmp = |
--- |
| 3201 |
DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()), |
0 |
3201 |
DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()), |
0 |
| 3202 |
LoOps[0], LoOps[1], ISD::SETULT); |
--- |
3202 |
LoOps[0], LoOps[1], ISD::SETULT); |
--- |
| 3203 |
|
--- |
3203 |
|
--- |
| 3204 |
SDValue Borrow; |
0 |
3204 |
SDValue Borrow; |
0 |
| 3205 |
if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) |
0 |
3205 |
if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) |
0 |
| 3206 |
Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT); |
0 |
3206 |
Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT); |
0 |
| 3207 |
else |
--- |
3207 |
else |
--- |
| 3208 |
Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT), |
0 |
3208 |
Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT), |
0 |
| 3209 |
DAG.getConstant(0, dl, NVT)); |
0 |
3209 |
DAG.getConstant(0, dl, NVT)); |
0 |
| 3210 |
|
--- |
3210 |
|
--- |
| 3211 |
Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow); |
0 |
3211 |
Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow); |
0 |
| 3212 |
} |
--- |
3212 |
} |
--- |
| 3213 |
} |
0 |
3213 |
} |
0 |
| 3214 |
|
--- |
3214 |
|
--- |
| 3215 |
void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N, |
0 |
3215 |
void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N, |
0 |
| 3216 |
SDValue &Lo, SDValue &Hi) { |
--- |
3216 |
SDValue &Lo, SDValue &Hi) { |
--- |
| 3217 |
// Expand the subcomponents. |
--- |
3217 |
// Expand the subcomponents. |
--- |
| 3218 |
SDValue LHSL, LHSH, RHSL, RHSH; |
0 |
3218 |
SDValue LHSL, LHSH, RHSL, RHSH; |
0 |
| 3219 |
SDLoc dl(N); |
0 |
3219 |
SDLoc dl(N); |
0 |
| 3220 |
GetExpandedInteger(N->getOperand(0), LHSL, LHSH); |
0 |
3220 |
GetExpandedInteger(N->getOperand(0), LHSL, LHSH); |
0 |
| 3221 |
GetExpandedInteger(N->getOperand(1), RHSL, RHSH); |
0 |
3221 |
GetExpandedInteger(N->getOperand(1), RHSL, RHSH); |
0 |
| 3222 |
SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); |
0 |
3222 |
SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); |
0 |
| 3223 |
SDValue LoOps[2] = { LHSL, RHSL }; |
0 |
3223 |
SDValue LoOps[2] = { LHSL, RHSL }; |
0 |
| 3224 |
SDValue HiOps[3] = { LHSH, RHSH }; |
0 |
3224 |
SDValue HiOps[3] = { LHSH, RHSH }; |
0 |
| 3225 |
|
--- |
3225 |
|
--- |
| 3226 |
if (N->getOpcode() == ISD::ADDC) { |
0 |
3226 |
if (N->getOpcode() == ISD::ADDC) { |
0 |
| 3227 |
Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps); |
0 |
3227 |
Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps); |
0 |
| 3228 |
HiOps[2] = Lo.getValue(1); |
0 |
3228 |
HiOps[2] = Lo.getValue(1); |
0 |
| 3229 |
Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps); |
0 |
3229 |
Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps); |
0 |
| 3230 |
} else { |
--- |
3230 |
} else { |
--- |
| 3231 |
Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps); |
0 |
3231 |
Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps); |
0 |
| 3232 |
HiOps[2] = Lo.getValue(1); |
0 |
3232 |
HiOps[2] = Lo.getValue(1); |
0 |
| 3233 |
Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps); |
0 |
3233 |
Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps); |
0 |
| 3234 |
} |
--- |
3234 |
} |
--- |
| 3235 |
|
--- |
3235 |
|
--- |
| 3236 |
// Legalized the flag result - switch anything that used the old flag to |
--- |
3236 |
// Legalized the flag result - switch anything that used the old flag to |
--- |
| 3237 |
// use the new one. |
--- |
3237 |
// use the new one. |
--- |
| 3238 |
ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); |
0 |
3238 |
ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); |
0 |
| 3239 |
} |
0 |
3239 |
} |
0 |
| 3240 |
|
--- |
3240 |
|
--- |
| 3241 |
void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N, |
0 |
3241 |
void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N, |
0 |
| 3242 |
SDValue &Lo, SDValue &Hi) { |
--- |
3242 |
SDValue &Lo, SDValue &Hi) { |
--- |
| 3243 |
// Expand the subcomponents. |
--- |
3243 |
// Expand the subcomponents. |
--- |
| 3244 |
SDValue LHSL, LHSH, RHSL, RHSH; |
0 |
3244 |
SDValue LHSL, LHSH, RHSL, RHSH; |
0 |
| 3245 |
SDLoc dl(N); |
0 |
3245 |
SDLoc dl(N); |
0 |
| 3246 |
GetExpandedInteger(N->getOperand(0), LHSL, LHSH); |
0 |
3246 |
GetExpandedInteger(N->getOperand(0), LHSL, LHSH); |
0 |
| 3247 |
GetExpandedInteger(N->getOperand(1), RHSL, RHSH); |
0 |
3247 |
GetExpandedInteger(N->getOperand(1), RHSL, RHSH); |
0 |
| 3248 |
SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); |
0 |
3248 |
SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); |
0 |
| 3249 |
SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) }; |
0 |
3249 |
SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) }; |
0 |
| 3250 |
SDValue HiOps[3] = { LHSH, RHSH }; |
0 |
3250 |
SDValue HiOps[3] = { LHSH, RHSH }; |
0 |
| 3251 |
|
--- |
3251 |
|
--- |
| 3252 |
Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); |
0 |
3252 |
Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); |
0 |
| 3253 |
HiOps[2] = Lo.getValue(1); |
0 |
3253 |
HiOps[2] = Lo.getValue(1); |
0 |
| 3254 |
Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps); |
0 |
3254 |
Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps); |
0 |
| 3255 |
|
--- |
3255 |
|
--- |
| 3256 |
// Legalized the flag result - switch anything that used the old flag to |
--- |
3256 |
// Legalized the flag result - switch anything that used the old flag to |
--- |
| 3257 |
// use the new one. |
--- |
3257 |
// use the new one. |
--- |
| 3258 |
ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); |
0 |
3258 |
ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); |
0 |
| 3259 |
} |
0 |
3259 |
} |
0 |
| 3260 |
|
--- |
3260 |
|
--- |
| 3261 |
void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N, |
0 |
3261 |
void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N, |
0 |
| 3262 |
SDValue &Lo, SDValue &Hi) { |
--- |
3262 |
SDValue &Lo, SDValue &Hi) { |
--- |
| 3263 |
SDValue LHS = N->getOperand(0); |
0 |
3263 |
SDValue LHS = N->getOperand(0); |
0 |
| 3264 |
SDValue RHS = N->getOperand(1); |
0 |
3264 |
SDValue RHS = N->getOperand(1); |
0 |
| 3265 |
SDLoc dl(N); |
0 |
3265 |
SDLoc dl(N); |
0 |
| 3266 |
|
--- |
3266 |
|
--- |
| 3267 |
SDValue Ovf; |
0 |
3267 |
SDValue Ovf; |
0 |
| 3268 |
|
--- |
3268 |
|
--- |
| 3269 |
unsigned CarryOp, NoCarryOp; |
--- |
3269 |
unsigned CarryOp, NoCarryOp; |
--- |
| 3270 |
ISD::CondCode Cond; |
--- |
3270 |
ISD::CondCode Cond; |
--- |
| 3271 |
switch(N->getOpcode()) { |
0 |
3271 |
switch(N->getOpcode()) { |
0 |
| 3272 |
case ISD::UADDO: |
0 |
3272 |
case ISD::UADDO: |
0 |
| 3273 |
CarryOp = ISD::UADDO_CARRY; |
0 |
3273 |
CarryOp = ISD::UADDO_CARRY; |
0 |
| 3274 |
NoCarryOp = ISD::ADD; |
0 |
3274 |
NoCarryOp = ISD::ADD; |
0 |
| 3275 |
Cond = ISD::SETULT; |
0 |
3275 |
Cond = ISD::SETULT; |
0 |
| 3276 |
break; |
0 |
3276 |
break; |
0 |
| 3277 |
case ISD::USUBO: |
0 |
3277 |
case ISD::USUBO: |
0 |
| 3278 |
CarryOp = ISD::USUBO_CARRY; |
0 |
3278 |
CarryOp = ISD::USUBO_CARRY; |
0 |
| 3279 |
NoCarryOp = ISD::SUB; |
0 |
3279 |
NoCarryOp = ISD::SUB; |
0 |
| 3280 |
Cond = ISD::SETUGT; |
0 |
3280 |
Cond = ISD::SETUGT; |
0 |
| 3281 |
break; |
0 |
3281 |
break; |
0 |
| 3282 |
default: |
0 |
3282 |
default: |
0 |
| 3283 |
llvm_unreachable("Node has unexpected Opcode"); |
0 |
3283 |
llvm_unreachable("Node has unexpected Opcode"); |
0 |
| 3284 |
} |
--- |
3284 |
} |
--- |
| 3285 |
|
--- |
3285 |
|
--- |
| 3286 |
bool HasCarryOp = TLI.isOperationLegalOrCustom( |
0 |
3286 |
bool HasCarryOp = TLI.isOperationLegalOrCustom( |
0 |
| 3287 |
CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType())); |
0 |
3287 |
CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType())); |
0 |
| 3288 |
|
--- |
3288 |
|
--- |
| 3289 |
if (HasCarryOp) { |
0 |
3289 |
if (HasCarryOp) { |
0 |
| 3290 |
// Expand the subcomponents. |
--- |
3290 |
// Expand the subcomponents. |
--- |
| 3291 |
SDValue LHSL, LHSH, RHSL, RHSH; |
0 |
3291 |
SDValue LHSL, LHSH, RHSL, RHSH; |
0 |
| 3292 |
GetExpandedInteger(LHS, LHSL, LHSH); |
0 |
3292 |
GetExpandedInteger(LHS, LHSL, LHSH); |
0 |
| 3293 |
GetExpandedInteger(RHS, RHSL, RHSH); |
0 |
3293 |
GetExpandedInteger(RHS, RHSL, RHSH); |
0 |
| 3294 |
SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1)); |
0 |
3294 |
SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1)); |
0 |
| 3295 |
SDValue LoOps[2] = { LHSL, RHSL }; |
0 |
3295 |
SDValue LoOps[2] = { LHSL, RHSL }; |
0 |
| 3296 |
SDValue HiOps[3] = { LHSH, RHSH }; |
0 |
3296 |
SDValue HiOps[3] = { LHSH, RHSH }; |
0 |
| 3297 |
|
--- |
3297 |
|
--- |
| 3298 |
Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); |
0 |
3298 |
Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); |
0 |
| 3299 |
HiOps[2] = Lo.getValue(1); |
0 |
3299 |
HiOps[2] = Lo.getValue(1); |
0 |
| 3300 |
Hi = DAG.getNode(CarryOp, dl, VTList, HiOps); |
0 |
3300 |
Hi = DAG.getNode(CarryOp, dl, VTList, HiOps); |
0 |
| 3301 |
|
--- |
3301 |
|
--- |
| 3302 |
Ovf = Hi.getValue(1); |
0 |
3302 |
Ovf = Hi.getValue(1); |
0 |
| 3303 |
} else { |
--- |
3303 |
} else { |
--- |
| 3304 |
// Expand the result by simply replacing it with the equivalent |
--- |
3304 |
// Expand the result by simply replacing it with the equivalent |
--- |
| 3305 |
// non-overflow-checking operation. |
--- |
3305 |
// non-overflow-checking operation. |
--- |
| 3306 |
SDValue Sum = DAG.getNode(NoCarryOp, dl, LHS.getValueType(), LHS, RHS); |
0 |
3306 |
SDValue Sum = DAG.getNode(NoCarryOp, dl, LHS.getValueType(), LHS, RHS); |
0 |
| 3307 |
SplitInteger(Sum, Lo, Hi); |
0 |
3307 |
SplitInteger(Sum, Lo, Hi); |
0 |
| 3308 |
|
--- |
3308 |
|
--- |
| 3309 |
if (N->getOpcode() == ISD::UADDO && isOneConstant(RHS)) { |
0 |
3309 |
if (N->getOpcode() == ISD::UADDO && isOneConstant(RHS)) { |
0 |
| 3310 |
// Special case: uaddo X, 1 overflowed if X+1 == 0. We can detect this |
--- |
3310 |
// Special case: uaddo X, 1 overflowed if X+1 == 0. We can detect this |
--- |
| 3311 |
// with (Lo | Hi) == 0. |
--- |
3311 |
// with (Lo | Hi) == 0. |
--- |
| 3312 |
SDValue Or = DAG.getNode(ISD::OR, dl, Lo.getValueType(), Lo, Hi); |
0 |
3312 |
SDValue Or = DAG.getNode(ISD::OR, dl, Lo.getValueType(), Lo, Hi); |
0 |
| 3313 |
Ovf = DAG.getSetCC(dl, N->getValueType(1), Or, |
0 |
3313 |
Ovf = DAG.getSetCC(dl, N->getValueType(1), Or, |
0 |
| 3314 |
DAG.getConstant(0, dl, Lo.getValueType()), ISD::SETEQ); |
0 |
3314 |
DAG.getConstant(0, dl, Lo.getValueType()), ISD::SETEQ); |
0 |
| 3315 |
} else if (N->getOpcode() == ISD::UADDO && isAllOnesConstant(RHS)) { |
0 |
3315 |
} else if (N->getOpcode() == ISD::UADDO && isAllOnesConstant(RHS)) { |
0 |
| 3316 |
// Special case: uaddo X, -1 overflows if X == 0. |
--- |
3316 |
// Special case: uaddo X, -1 overflows if X == 0. |
--- |
| 3317 |
Ovf = |
0 |
3317 |
Ovf = |
0 |
| 3318 |
DAG.getSetCC(dl, N->getValueType(1), LHS, |
0 |
3318 |
DAG.getSetCC(dl, N->getValueType(1), LHS, |
0 |
| 3319 |
DAG.getConstant(0, dl, LHS.getValueType()), ISD::SETNE); |
0 |
3319 |
DAG.getConstant(0, dl, LHS.getValueType()), ISD::SETNE); |
0 |
| 3320 |
} else { |
--- |
3320 |
} else { |
--- |
| 3321 |
// Calculate the overflow: addition overflows iff a + b < a, and |
--- |
3321 |
// Calculate the overflow: addition overflows iff a + b < a, and |
--- |
| 3322 |
// subtraction overflows iff a - b > a. |
--- |
3322 |
// subtraction overflows iff a - b > a. |
--- |
| 3323 |
Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond); |
0 |
3323 |
Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond); |
0 |
| 3324 |
} |
--- |
3324 |
} |
--- |
| 3325 |
} |
--- |
3325 |
} |
--- |
| 3326 |
|
--- |
3326 |
|
--- |
| 3327 |
// Legalized the flag result - switch anything that used the old flag to |
--- |
3327 |
// Legalized the flag result - switch anything that used the old flag to |
--- |
| 3328 |
// use the new one. |
--- |
3328 |
// use the new one. |
--- |
| 3329 |
ReplaceValueWith(SDValue(N, 1), Ovf); |
0 |
3329 |
ReplaceValueWith(SDValue(N, 1), Ovf); |
0 |
| 3330 |
} |
0 |
3330 |
} |
0 |
| 3331 |
|
--- |
3331 |
|
--- |
| 3332 |
void DAGTypeLegalizer::ExpandIntRes_UADDSUBO_CARRY(SDNode *N, SDValue &Lo, |
0 |
3332 |
void DAGTypeLegalizer::ExpandIntRes_UADDSUBO_CARRY(SDNode *N, SDValue &Lo, |
0 |
| 3333 |
SDValue &Hi) { |
--- |
3333 |
SDValue &Hi) { |
--- |
| 3334 |
// Expand the subcomponents. |
--- |
3334 |
// Expand the subcomponents. |
--- |
| 3335 |
SDValue LHSL, LHSH, RHSL, RHSH; |
0 |
3335 |
SDValue LHSL, LHSH, RHSL, RHSH; |
0 |
| 3336 |
SDLoc dl(N); |
0 |
3336 |
SDLoc dl(N); |
0 |
| 3337 |
GetExpandedInteger(N->getOperand(0), LHSL, LHSH); |
0 |
3337 |
GetExpandedInteger(N->getOperand(0), LHSL, LHSH); |
0 |
| 3338 |
GetExpandedInteger(N->getOperand(1), RHSL, RHSH); |
0 |
3338 |
GetExpandedInteger(N->getOperand(1), RHSL, RHSH); |
0 |
| 3339 |
SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1)); |
0 |
3339 |
SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1)); |
0 |
| 3340 |
SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) }; |
0 |
3340 |
SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) }; |
0 |
| 3341 |
SDValue HiOps[3] = { LHSH, RHSH, SDValue() }; |
0 |
3341 |
SDValue HiOps[3] = { LHSH, RHSH, SDValue() }; |
0 |
| 3342 |
|
--- |
3342 |
|
--- |
| 3343 |
Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); |
0 |
3343 |
Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); |
0 |
| 3344 |
HiOps[2] = Lo.getValue(1); |
0 |
3344 |
HiOps[2] = Lo.getValue(1); |
0 |
| 3345 |
Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps); |
0 |
3345 |
Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps); |
0 |
| 3346 |
|
--- |
3346 |
|
--- |
| 3347 |
// Legalized the flag result - switch anything that used the old flag to |
--- |
3347 |
// Legalized the flag result - switch anything that used the old flag to |
--- |
| 3348 |
// use the new one. |
--- |
3348 |
// use the new one. |
--- |
| 3349 |
ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); |
0 |
3349 |
ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); |
0 |
| 3350 |
} |
0 |
3350 |
} |
0 |
| 3351 |
|
--- |
3351 |
|
--- |
| 3352 |
void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(SDNode *N, |
0 |
3352 |
void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(SDNode *N, |
0 |
| 3353 |
SDValue &Lo, SDValue &Hi) { |
--- |
3353 |
SDValue &Lo, SDValue &Hi) { |
--- |
| 3354 |
// Expand the subcomponents. |
--- |
3354 |
// Expand the subcomponents. |
--- |
| 3355 |
SDValue LHSL, LHSH, RHSL, RHSH; |
0 |
3355 |
SDValue LHSL, LHSH, RHSL, RHSH; |
0 |
| 3356 |
SDLoc dl(N); |
0 |
3356 |
SDLoc dl(N); |
0 |
| 3357 |
GetExpandedInteger(N->getOperand(0), LHSL, LHSH); |
0 |
3357 |
GetExpandedInteger(N->getOperand(0), LHSL, LHSH); |
0 |
| 3358 |
GetExpandedInteger(N->getOperand(1), RHSL, RHSH); |
0 |
3358 |
GetExpandedInteger(N->getOperand(1), RHSL, RHSH); |
0 |
| 3359 |
SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1)); |
0 |
3359 |
SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1)); |
0 |
| 3360 |
|
--- |
3360 |
|
--- |
| 3361 |
// We need to use an unsigned carry op for the lo part. |
--- |
3361 |
// We need to use an unsigned carry op for the lo part. |
--- |
| 3362 |
unsigned CarryOp = |
--- |
3362 |
unsigned CarryOp = |
--- |
| 3363 |
N->getOpcode() == ISD::SADDO_CARRY ? ISD::UADDO_CARRY : ISD::USUBO_CARRY; |
0 |
3363 |
N->getOpcode() == ISD::SADDO_CARRY ? ISD::UADDO_CARRY : ISD::USUBO_CARRY; |
0 |
| 3364 |
Lo = DAG.getNode(CarryOp, dl, VTList, { LHSL, RHSL, N->getOperand(2) }); |
0 |
3364 |
Lo = DAG.getNode(CarryOp, dl, VTList, { LHSL, RHSL, N->getOperand(2) }); |
0 |
| 3365 |
Hi = DAG.getNode(N->getOpcode(), dl, VTList, { LHSH, RHSH, Lo.getValue(1) }); |
0 |
3365 |
Hi = DAG.getNode(N->getOpcode(), dl, VTList, { LHSH, RHSH, Lo.getValue(1) }); |
0 |
| 3366 |
|
--- |
3366 |
|
--- |
| 3367 |
// Legalized the flag result - switch anything that used the old flag to |
--- |
3367 |
// Legalized the flag result - switch anything that used the old flag to |
--- |
| 3368 |
// use the new one. |
--- |
3368 |
// use the new one. |
--- |
| 3369 |
ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); |
0 |
3369 |
ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); |
0 |
| 3370 |
} |
0 |
3370 |
} |
0 |
| 3371 |
|
--- |
3371 |
|
--- |
| 3372 |
void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N, |
0 |
3372 |
void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N, |
0 |
| 3373 |
SDValue &Lo, SDValue &Hi) { |
--- |
3373 |
SDValue &Lo, SDValue &Hi) { |
--- |
| 3374 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
3374 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
| 3375 |
SDLoc dl(N); |
0 |
3375 |
SDLoc dl(N); |
0 |
| 3376 |
SDValue Op = N->getOperand(0); |
0 |
3376 |
SDValue Op = N->getOperand(0); |
0 |
| 3377 |
if (Op.getValueType().bitsLE(NVT)) { |
0 |
3377 |
if (Op.getValueType().bitsLE(NVT)) { |
0 |
| 3378 |
// The low part is any extension of the input (which degenerates to a copy). |
--- |
3378 |
// The low part is any extension of the input (which degenerates to a copy). |
--- |
| 3379 |
Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op); |
0 |
3379 |
Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op); |
0 |
| 3380 |
Hi = DAG.getUNDEF(NVT); // The high part is undefined. |
0 |
3380 |
Hi = DAG.getUNDEF(NVT); // The high part is undefined. |
0 |
| 3381 |
} else { |
--- |
3381 |
} else { |
--- |
| 3382 |
// For example, extension of an i48 to an i64. The operand type necessarily |
--- |
3382 |
// For example, extension of an i48 to an i64. The operand type necessarily |
--- |
| 3383 |
// promotes to the result type, so will end up being expanded too. |
--- |
3383 |
// promotes to the result type, so will end up being expanded too. |
--- |
| 3384 |
assert(getTypeAction(Op.getValueType()) == |
0 |
3384 |
assert(getTypeAction(Op.getValueType()) == |
0 |
| 3385 |
TargetLowering::TypePromoteInteger && |
--- |
3385 |
TargetLowering::TypePromoteInteger && |
--- |
| 3386 |
"Only know how to promote this result!"); |
--- |
3386 |
"Only know how to promote this result!"); |
--- |
| 3387 |
SDValue Res = GetPromotedInteger(Op); |
0 |
3387 |
SDValue Res = GetPromotedInteger(Op); |
0 |
| 3388 |
assert(Res.getValueType() == N->getValueType(0) && |
0 |
3388 |
assert(Res.getValueType() == N->getValueType(0) && |
0 |
| 3389 |
"Operand over promoted?"); |
--- |
3389 |
"Operand over promoted?"); |
--- |
| 3390 |
// Split the promoted operand. This will simplify when it is expanded. |
--- |
3390 |
// Split the promoted operand. This will simplify when it is expanded. |
--- |
| 3391 |
SplitInteger(Res, Lo, Hi); |
0 |
3391 |
SplitInteger(Res, Lo, Hi); |
0 |
| 3392 |
} |
--- |
3392 |
} |
--- |
| 3393 |
} |
0 |
3393 |
} |
0 |
| 3394 |
|
--- |
3394 |
|
--- |
| 3395 |
void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N, |
0 |
3395 |
void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N, |
0 |
| 3396 |
SDValue &Lo, SDValue &Hi) { |
--- |
3396 |
SDValue &Lo, SDValue &Hi) { |
--- |
| 3397 |
SDLoc dl(N); |
0 |
3397 |
SDLoc dl(N); |
0 |
| 3398 |
GetExpandedInteger(N->getOperand(0), Lo, Hi); |
0 |
3398 |
GetExpandedInteger(N->getOperand(0), Lo, Hi); |
0 |
| 3399 |
EVT NVT = Lo.getValueType(); |
0 |
3399 |
EVT NVT = Lo.getValueType(); |
0 |
| 3400 |
EVT EVT = cast(N->getOperand(1))->getVT(); |
0 |
3400 |
EVT EVT = cast(N->getOperand(1))->getVT(); |
0 |
| 3401 |
unsigned NVTBits = NVT.getSizeInBits(); |
0 |
3401 |
unsigned NVTBits = NVT.getSizeInBits(); |
0 |
| 3402 |
unsigned EVTBits = EVT.getSizeInBits(); |
0 |
3402 |
unsigned EVTBits = EVT.getSizeInBits(); |
0 |
| 3403 |
|
--- |
3403 |
|
--- |
| 3404 |
if (NVTBits < EVTBits) { |
0 |
3404 |
if (NVTBits < EVTBits) { |
0 |
| 3405 |
Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi, |
0 |
3405 |
Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi, |
0 |
| 3406 |
DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), |
0 |
3406 |
DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), |
0 |
| 3407 |
EVTBits - NVTBits))); |
--- |
3407 |
EVTBits - NVTBits))); |
--- |
| 3408 |
} else { |
--- |
3408 |
} else { |
--- |
| 3409 |
Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT)); |
0 |
3409 |
Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT)); |
0 |
| 3410 |
// The high part replicates the sign bit of Lo, make it explicit. |
--- |
3410 |
// The high part replicates the sign bit of Lo, make it explicit. |
--- |
| 3411 |
Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, |
0 |
3411 |
Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, |
0 |
| 3412 |
DAG.getConstant(NVTBits - 1, dl, |
0 |
3412 |
DAG.getConstant(NVTBits - 1, dl, |
0 |
| 3413 |
TLI.getPointerTy(DAG.getDataLayout()))); |
0 |
3413 |
TLI.getPointerTy(DAG.getDataLayout()))); |
0 |
| 3414 |
} |
--- |
3414 |
} |
--- |
| 3415 |
} |
0 |
3415 |
} |
0 |
| 3416 |
|
--- |
3416 |
|
--- |
| 3417 |
void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N, |
0 |
3417 |
void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N, |
0 |
| 3418 |
SDValue &Lo, SDValue &Hi) { |
--- |
3418 |
SDValue &Lo, SDValue &Hi) { |
--- |
| 3419 |
SDLoc dl(N); |
0 |
3419 |
SDLoc dl(N); |
0 |
| 3420 |
GetExpandedInteger(N->getOperand(0), Lo, Hi); |
0 |
3420 |
GetExpandedInteger(N->getOperand(0), Lo, Hi); |
0 |
| 3421 |
EVT NVT = Lo.getValueType(); |
0 |
3421 |
EVT NVT = Lo.getValueType(); |
0 |
| 3422 |
EVT EVT = cast(N->getOperand(1))->getVT(); |
0 |
3422 |
EVT EVT = cast(N->getOperand(1))->getVT(); |
0 |
| 3423 |
unsigned NVTBits = NVT.getSizeInBits(); |
0 |
3423 |
unsigned NVTBits = NVT.getSizeInBits(); |
0 |
| 3424 |
unsigned EVTBits = EVT.getSizeInBits(); |
0 |
3424 |
unsigned EVTBits = EVT.getSizeInBits(); |
0 |
| 3425 |
|
--- |
3425 |
|
--- |
| 3426 |
if (NVTBits < EVTBits) { |
0 |
3426 |
if (NVTBits < EVTBits) { |
0 |
| 3427 |
Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi, |
0 |
3427 |
Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi, |
0 |
| 3428 |
DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), |
0 |
3428 |
DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), |
0 |
| 3429 |
EVTBits - NVTBits))); |
--- |
3429 |
EVTBits - NVTBits))); |
--- |
| 3430 |
} else { |
--- |
3430 |
} else { |
--- |
| 3431 |
Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT)); |
0 |
3431 |
Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT)); |
0 |
| 3432 |
// The high part must be zero, make it explicit. |
--- |
3432 |
// The high part must be zero, make it explicit. |
--- |
| 3433 |
Hi = DAG.getConstant(0, dl, NVT); |
0 |
3433 |
Hi = DAG.getConstant(0, dl, NVT); |
0 |
| 3434 |
} |
--- |
3434 |
} |
--- |
| 3435 |
} |
0 |
3435 |
} |
0 |
| 3436 |
|
--- |
3436 |
|
--- |
| 3437 |
void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N, |
0 |
3437 |
void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N, |
0 |
| 3438 |
SDValue &Lo, SDValue &Hi) { |
--- |
3438 |
SDValue &Lo, SDValue &Hi) { |
--- |
| 3439 |
SDLoc dl(N); |
0 |
3439 |
SDLoc dl(N); |
0 |
| 3440 |
GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands. |
0 |
3440 |
GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands. |
0 |
| 3441 |
Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo); |
0 |
3441 |
Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo); |
0 |
| 3442 |
Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi); |
0 |
3442 |
Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi); |
0 |
| 3443 |
} |
0 |
3443 |
} |
0 |
| 3444 |
|
--- |
3444 |
|
--- |
| 3445 |
void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N, |
0 |
3445 |
void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N, |
0 |
| 3446 |
SDValue &Lo, SDValue &Hi) { |
--- |
3446 |
SDValue &Lo, SDValue &Hi) { |
--- |
| 3447 |
SDLoc dl(N); |
0 |
3447 |
SDLoc dl(N); |
0 |
| 3448 |
GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands. |
0 |
3448 |
GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands. |
0 |
| 3449 |
Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo); |
0 |
3449 |
Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo); |
0 |
| 3450 |
Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi); |
0 |
3450 |
Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi); |
0 |
| 3451 |
} |
0 |
3451 |
} |
0 |
| 3452 |
|
--- |
3452 |
|
--- |
| 3453 |
void DAGTypeLegalizer::ExpandIntRes_PARITY(SDNode *N, SDValue &Lo, |
0 |
3453 |
void DAGTypeLegalizer::ExpandIntRes_PARITY(SDNode *N, SDValue &Lo, |
0 |
| 3454 |
SDValue &Hi) { |
--- |
3454 |
SDValue &Hi) { |
--- |
| 3455 |
SDLoc dl(N); |
0 |
3455 |
SDLoc dl(N); |
0 |
| 3456 |
// parity(HiLo) -> parity(Lo^Hi) |
--- |
3456 |
// parity(HiLo) -> parity(Lo^Hi) |
--- |
| 3457 |
GetExpandedInteger(N->getOperand(0), Lo, Hi); |
0 |
3457 |
GetExpandedInteger(N->getOperand(0), Lo, Hi); |
0 |
| 3458 |
EVT NVT = Lo.getValueType(); |
0 |
3458 |
EVT NVT = Lo.getValueType(); |
0 |
| 3459 |
Lo = |
0 |
3459 |
Lo = |
0 |
| 3460 |
DAG.getNode(ISD::PARITY, dl, NVT, DAG.getNode(ISD::XOR, dl, NVT, Lo, Hi)); |
0 |
3460 |
DAG.getNode(ISD::PARITY, dl, NVT, DAG.getNode(ISD::XOR, dl, NVT, Lo, Hi)); |
0 |
| 3461 |
Hi = DAG.getConstant(0, dl, NVT); |
0 |
3461 |
Hi = DAG.getConstant(0, dl, NVT); |
0 |
| 3462 |
} |
0 |
3462 |
} |
0 |
| 3463 |
|
--- |
3463 |
|
--- |
| 3464 |
void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N, |
0 |
3464 |
void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N, |
0 |
| 3465 |
SDValue &Lo, SDValue &Hi) { |
--- |
3465 |
SDValue &Lo, SDValue &Hi) { |
--- |
| 3466 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
3466 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
| 3467 |
unsigned NBitWidth = NVT.getSizeInBits(); |
0 |
3467 |
unsigned NBitWidth = NVT.getSizeInBits(); |
0 |
| 3468 |
auto Constant = cast(N); |
0 |
3468 |
auto Constant = cast(N); |
0 |
| 3469 |
const APInt &Cst = Constant->getAPIntValue(); |
0 |
3469 |
const APInt &Cst = Constant->getAPIntValue(); |
0 |
| 3470 |
bool IsTarget = Constant->isTargetOpcode(); |
0 |
3470 |
bool IsTarget = Constant->isTargetOpcode(); |
0 |
| 3471 |
bool IsOpaque = Constant->isOpaque(); |
0 |
3471 |
bool IsOpaque = Constant->isOpaque(); |
0 |
| 3472 |
SDLoc dl(N); |
0 |
3472 |
SDLoc dl(N); |
0 |
| 3473 |
Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque); |
0 |
3473 |
Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque); |
0 |
| 3474 |
Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget, |
0 |
3474 |
Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget, |
0 |
| 3475 |
IsOpaque); |
--- |
3475 |
IsOpaque); |
--- |
| 3476 |
} |
0 |
3476 |
} |
0 |
| 3477 |
|
--- |
3477 |
|
--- |
| 3478 |
void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) { |
0 |
3478 |
void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) { |
0 |
| 3479 |
SDLoc dl(N); |
0 |
3479 |
SDLoc dl(N); |
0 |
| 3480 |
|
--- |
3480 |
|
--- |
| 3481 |
SDValue N0 = N->getOperand(0); |
0 |
3481 |
SDValue N0 = N->getOperand(0); |
0 |
| 3482 |
GetExpandedInteger(N0, Lo, Hi); |
0 |
3482 |
GetExpandedInteger(N0, Lo, Hi); |
0 |
| 3483 |
EVT NVT = Lo.getValueType(); |
0 |
3483 |
EVT NVT = Lo.getValueType(); |
0 |
| 3484 |
|
--- |
3484 |
|
--- |
| 3485 |
// If the upper half is all sign bits, then we can perform the ABS on the |
--- |
3485 |
// If the upper half is all sign bits, then we can perform the ABS on the |
--- |
| 3486 |
// lower half and zero-extend. |
--- |
3486 |
// lower half and zero-extend. |
--- |
| 3487 |
if (DAG.ComputeNumSignBits(N0) > NVT.getScalarSizeInBits()) { |
0 |
3487 |
if (DAG.ComputeNumSignBits(N0) > NVT.getScalarSizeInBits()) { |
0 |
| 3488 |
Lo = DAG.getNode(ISD::ABS, dl, NVT, Lo); |
0 |
3488 |
Lo = DAG.getNode(ISD::ABS, dl, NVT, Lo); |
0 |
| 3489 |
Hi = DAG.getConstant(0, dl, NVT); |
0 |
3489 |
Hi = DAG.getConstant(0, dl, NVT); |
0 |
| 3490 |
return; |
0 |
3490 |
return; |
0 |
| 3491 |
} |
--- |
3491 |
} |
--- |
| 3492 |
|
--- |
3492 |
|
--- |
| 3493 |
// If we have USUBO_CARRY, use the expanded form of the sra+xor+sub sequence |
--- |
3493 |
// If we have USUBO_CARRY, use the expanded form of the sra+xor+sub sequence |
--- |
| 3494 |
// we use in LegalizeDAG. The SUB part of the expansion is based on |
--- |
3494 |
// we use in LegalizeDAG. The SUB part of the expansion is based on |
--- |
| 3495 |
// ExpandIntRes_ADDSUB which also uses USUBO_CARRY/USUBO after checking that |
--- |
3495 |
// ExpandIntRes_ADDSUB which also uses USUBO_CARRY/USUBO after checking that |
--- |
| 3496 |
// USUBO_CARRY is LegalOrCustom. Each of the pieces here can be further |
--- |
3496 |
// USUBO_CARRY is LegalOrCustom. Each of the pieces here can be further |
--- |
| 3497 |
// expanded if needed. Shift expansion has a special case for filling with |
--- |
3497 |
// expanded if needed. Shift expansion has a special case for filling with |
--- |
| 3498 |
// sign bits so that we will only end up with one SRA. |
--- |
3498 |
// sign bits so that we will only end up with one SRA. |
--- |
| 3499 |
bool HasSubCarry = TLI.isOperationLegalOrCustom( |
0 |
3499 |
bool HasSubCarry = TLI.isOperationLegalOrCustom( |
0 |
| 3500 |
ISD::USUBO_CARRY, TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); |
0 |
3500 |
ISD::USUBO_CARRY, TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); |
0 |
| 3501 |
if (HasSubCarry) { |
0 |
3501 |
if (HasSubCarry) { |
0 |
| 3502 |
SDValue Sign = DAG.getNode( |
0 |
3502 |
SDValue Sign = DAG.getNode( |
0 |
| 3503 |
ISD::SRA, dl, NVT, Hi, |
--- |
3503 |
ISD::SRA, dl, NVT, Hi, |
--- |
| 3504 |
DAG.getShiftAmountConstant(NVT.getSizeInBits() - 1, NVT, dl)); |
0 |
3504 |
DAG.getShiftAmountConstant(NVT.getSizeInBits() - 1, NVT, dl)); |
0 |
| 3505 |
SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT)); |
0 |
3505 |
SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT)); |
0 |
| 3506 |
Lo = DAG.getNode(ISD::XOR, dl, NVT, Lo, Sign); |
0 |
3506 |
Lo = DAG.getNode(ISD::XOR, dl, NVT, Lo, Sign); |
0 |
| 3507 |
Hi = DAG.getNode(ISD::XOR, dl, NVT, Hi, Sign); |
0 |
3507 |
Hi = DAG.getNode(ISD::XOR, dl, NVT, Hi, Sign); |
0 |
| 3508 |
Lo = DAG.getNode(ISD::USUBO, dl, VTList, Lo, Sign); |
0 |
3508 |
Lo = DAG.getNode(ISD::USUBO, dl, VTList, Lo, Sign); |
0 |
| 3509 |
Hi = DAG.getNode(ISD::USUBO_CARRY, dl, VTList, Hi, Sign, Lo.getValue(1)); |
0 |
3509 |
Hi = DAG.getNode(ISD::USUBO_CARRY, dl, VTList, Hi, Sign, Lo.getValue(1)); |
0 |
| 3510 |
return; |
0 |
3510 |
return; |
0 |
| 3511 |
} |
--- |
3511 |
} |
--- |
| 3512 |
|
--- |
3512 |
|
--- |
| 3513 |
// abs(HiLo) -> (Hi < 0 ? -HiLo : HiLo) |
--- |
3513 |
// abs(HiLo) -> (Hi < 0 ? -HiLo : HiLo) |
--- |
| 3514 |
EVT VT = N->getValueType(0); |
0 |
3514 |
EVT VT = N->getValueType(0); |
0 |
| 3515 |
SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, |
0 |
3515 |
SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, |
0 |
| 3516 |
DAG.getConstant(0, dl, VT), N0); |
0 |
3516 |
DAG.getConstant(0, dl, VT), N0); |
0 |
| 3517 |
SDValue NegLo, NegHi; |
0 |
3517 |
SDValue NegLo, NegHi; |
0 |
| 3518 |
SplitInteger(Neg, NegLo, NegHi); |
0 |
3518 |
SplitInteger(Neg, NegLo, NegHi); |
0 |
| 3519 |
|
--- |
3519 |
|
--- |
| 3520 |
SDValue HiIsNeg = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi, |
0 |
3520 |
SDValue HiIsNeg = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi, |
0 |
| 3521 |
DAG.getConstant(0, dl, NVT), ISD::SETLT); |
0 |
3521 |
DAG.getConstant(0, dl, NVT), ISD::SETLT); |
0 |
| 3522 |
Lo = DAG.getSelect(dl, NVT, HiIsNeg, NegLo, Lo); |
0 |
3522 |
Lo = DAG.getSelect(dl, NVT, HiIsNeg, NegLo, Lo); |
0 |
| 3523 |
Hi = DAG.getSelect(dl, NVT, HiIsNeg, NegHi, Hi); |
0 |
3523 |
Hi = DAG.getSelect(dl, NVT, HiIsNeg, NegHi, Hi); |
0 |
| 3524 |
} |
0 |
3524 |
} |
0 |
| 3525 |
|
--- |
3525 |
|
--- |
| 3526 |
void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N, |
0 |
3526 |
void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N, |
0 |
| 3527 |
SDValue &Lo, SDValue &Hi) { |
--- |
3527 |
SDValue &Lo, SDValue &Hi) { |
--- |
| 3528 |
SDLoc dl(N); |
0 |
3528 |
SDLoc dl(N); |
0 |
| 3529 |
// ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32) |
--- |
3529 |
// ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32) |
--- |
| 3530 |
GetExpandedInteger(N->getOperand(0), Lo, Hi); |
0 |
3530 |
GetExpandedInteger(N->getOperand(0), Lo, Hi); |
0 |
| 3531 |
EVT NVT = Lo.getValueType(); |
0 |
3531 |
EVT NVT = Lo.getValueType(); |
0 |
| 3532 |
|
--- |
3532 |
|
--- |
| 3533 |
SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi, |
0 |
3533 |
SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi, |
0 |
| 3534 |
DAG.getConstant(0, dl, NVT), ISD::SETNE); |
0 |
3534 |
DAG.getConstant(0, dl, NVT), ISD::SETNE); |
0 |
| 3535 |
|
--- |
3535 |
|
--- |
| 3536 |
SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo); |
0 |
3536 |
SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo); |
0 |
| 3537 |
SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi); |
0 |
3537 |
SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi); |
0 |
| 3538 |
|
--- |
3538 |
|
--- |
| 3539 |
Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ, |
0 |
3539 |
Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ, |
0 |
| 3540 |
DAG.getNode(ISD::ADD, dl, NVT, LoLZ, |
0 |
3540 |
DAG.getNode(ISD::ADD, dl, NVT, LoLZ, |
0 |
| 3541 |
DAG.getConstant(NVT.getSizeInBits(), dl, |
0 |
3541 |
DAG.getConstant(NVT.getSizeInBits(), dl, |
0 |
| 3542 |
NVT))); |
--- |
3542 |
NVT))); |
--- |
| 3543 |
Hi = DAG.getConstant(0, dl, NVT); |
0 |
3543 |
Hi = DAG.getConstant(0, dl, NVT); |
0 |
| 3544 |
} |
0 |
3544 |
} |
0 |
| 3545 |
|
--- |
3545 |
|
--- |
| 3546 |
void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N, |
0 |
3546 |
void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N, |
0 |
| 3547 |
SDValue &Lo, SDValue &Hi) { |
--- |
3547 |
SDValue &Lo, SDValue &Hi) { |
--- |
| 3548 |
SDLoc dl(N); |
0 |
3548 |
SDLoc dl(N); |
0 |
| 3549 |
// ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo) |
--- |
3549 |
// ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo) |
--- |
| 3550 |
GetExpandedInteger(N->getOperand(0), Lo, Hi); |
0 |
3550 |
GetExpandedInteger(N->getOperand(0), Lo, Hi); |
0 |
| 3551 |
EVT NVT = Lo.getValueType(); |
0 |
3551 |
EVT NVT = Lo.getValueType(); |
0 |
| 3552 |
Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo), |
0 |
3552 |
Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo), |
0 |
| 3553 |
DAG.getNode(ISD::CTPOP, dl, NVT, Hi)); |
0 |
3553 |
DAG.getNode(ISD::CTPOP, dl, NVT, Hi)); |
0 |
| 3554 |
Hi = DAG.getConstant(0, dl, NVT); |
0 |
3554 |
Hi = DAG.getConstant(0, dl, NVT); |
0 |
| 3555 |
} |
0 |
3555 |
} |
0 |
| 3556 |
|
--- |
3556 |
|
--- |
| 3557 |
void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N, |
0 |
3557 |
void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N, |
0 |
| 3558 |
SDValue &Lo, SDValue &Hi) { |
--- |
3558 |
SDValue &Lo, SDValue &Hi) { |
--- |
| 3559 |
SDLoc dl(N); |
0 |
3559 |
SDLoc dl(N); |
0 |
| 3560 |
// cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32) |
--- |
3560 |
// cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32) |
--- |
| 3561 |
GetExpandedInteger(N->getOperand(0), Lo, Hi); |
0 |
3561 |
GetExpandedInteger(N->getOperand(0), Lo, Hi); |
0 |
| 3562 |
EVT NVT = Lo.getValueType(); |
0 |
3562 |
EVT NVT = Lo.getValueType(); |
0 |
| 3563 |
|
--- |
3563 |
|
--- |
| 3564 |
SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, |
0 |
3564 |
SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, |
0 |
| 3565 |
DAG.getConstant(0, dl, NVT), ISD::SETNE); |
0 |
3565 |
DAG.getConstant(0, dl, NVT), ISD::SETNE); |
0 |
| 3566 |
|
--- |
3566 |
|
--- |
| 3567 |
SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo); |
0 |
3567 |
SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo); |
0 |
| 3568 |
SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi); |
0 |
3568 |
SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi); |
0 |
| 3569 |
|
--- |
3569 |
|
--- |
| 3570 |
Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ, |
0 |
3570 |
Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ, |
0 |
| 3571 |
DAG.getNode(ISD::ADD, dl, NVT, HiLZ, |
0 |
3571 |
DAG.getNode(ISD::ADD, dl, NVT, HiLZ, |
0 |
| 3572 |
DAG.getConstant(NVT.getSizeInBits(), dl, |
0 |
3572 |
DAG.getConstant(NVT.getSizeInBits(), dl, |
0 |
| 3573 |
NVT))); |
--- |
3573 |
NVT))); |
--- |
| 3574 |
Hi = DAG.getConstant(0, dl, NVT); |
0 |
3574 |
Hi = DAG.getConstant(0, dl, NVT); |
0 |
| 3575 |
} |
0 |
3575 |
} |
0 |
| 3576 |
|
--- |
3576 |
|
--- |
| 3577 |
void DAGTypeLegalizer::ExpandIntRes_GET_ROUNDING(SDNode *N, SDValue &Lo, |
0 |
3577 |
void DAGTypeLegalizer::ExpandIntRes_GET_ROUNDING(SDNode *N, SDValue &Lo, |
0 |
| 3578 |
SDValue &Hi) { |
--- |
3578 |
SDValue &Hi) { |
--- |
| 3579 |
SDLoc dl(N); |
0 |
3579 |
SDLoc dl(N); |
0 |
| 3580 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
3580 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
| 3581 |
unsigned NBitWidth = NVT.getSizeInBits(); |
0 |
3581 |
unsigned NBitWidth = NVT.getSizeInBits(); |
0 |
| 3582 |
|
--- |
3582 |
|
--- |
| 3583 |
Lo = DAG.getNode(ISD::GET_ROUNDING, dl, {NVT, MVT::Other}, N->getOperand(0)); |
0 |
3583 |
Lo = DAG.getNode(ISD::GET_ROUNDING, dl, {NVT, MVT::Other}, N->getOperand(0)); |
0 |
| 3584 |
SDValue Chain = Lo.getValue(1); |
0 |
3584 |
SDValue Chain = Lo.getValue(1); |
0 |
| 3585 |
// The high part is the sign of Lo, as -1 is a valid value for GET_ROUNDING |
--- |
3585 |
// The high part is the sign of Lo, as -1 is a valid value for GET_ROUNDING |
--- |
| 3586 |
Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, |
0 |
3586 |
Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, |
0 |
| 3587 |
DAG.getShiftAmountConstant(NBitWidth - 1, NVT, dl)); |
0 |
3587 |
DAG.getShiftAmountConstant(NBitWidth - 1, NVT, dl)); |
0 |
| 3588 |
|
--- |
3588 |
|
--- |
| 3589 |
// Legalize the chain result - switch anything that used the old chain to |
--- |
3589 |
// Legalize the chain result - switch anything that used the old chain to |
--- |
| 3590 |
// use the new one. |
--- |
3590 |
// use the new one. |
--- |
| 3591 |
ReplaceValueWith(SDValue(N, 1), Chain); |
0 |
3591 |
ReplaceValueWith(SDValue(N, 1), Chain); |
0 |
| 3592 |
} |
0 |
3592 |
} |
0 |
| 3593 |
|
--- |
3593 |
|
--- |
| 3594 |
void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo, |
0 |
3594 |
void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo, |
0 |
| 3595 |
SDValue &Hi) { |
--- |
3595 |
SDValue &Hi) { |
--- |
| 3596 |
SDLoc dl(N); |
0 |
3596 |
SDLoc dl(N); |
0 |
| 3597 |
EVT VT = N->getValueType(0); |
0 |
3597 |
EVT VT = N->getValueType(0); |
0 |
| 3598 |
|
--- |
3598 |
|
--- |
| 3599 |
bool IsStrict = N->isStrictFPOpcode(); |
0 |
3599 |
bool IsStrict = N->isStrictFPOpcode(); |
0 |
| 3600 |
SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); |
0 |
3600 |
SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); |
0 |
| 3601 |
SDValue Op = N->getOperand(IsStrict ? 1 : 0); |
0 |
3601 |
SDValue Op = N->getOperand(IsStrict ? 1 : 0); |
0 |
| 3602 |
if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) |
0 |
3602 |
if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) |
0 |
| 3603 |
Op = GetPromotedFloat(Op); |
0 |
3603 |
Op = GetPromotedFloat(Op); |
0 |
| 3604 |
|
--- |
3604 |
|
--- |
| 3605 |
if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) { |
0 |
3605 |
if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) { |
0 |
| 3606 |
EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()); |
0 |
3606 |
EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()); |
0 |
| 3607 |
Op = GetSoftPromotedHalf(Op); |
0 |
3607 |
Op = GetSoftPromotedHalf(Op); |
0 |
| 3608 |
Op = DAG.getNode(ISD::FP16_TO_FP, dl, NFPVT, Op); |
0 |
3608 |
Op = DAG.getNode(ISD::FP16_TO_FP, dl, NFPVT, Op); |
0 |
| 3609 |
Op = DAG.getNode(ISD::FP_TO_SINT, dl, VT, Op); |
0 |
3609 |
Op = DAG.getNode(ISD::FP_TO_SINT, dl, VT, Op); |
0 |
| 3610 |
SplitInteger(Op, Lo, Hi); |
0 |
3610 |
SplitInteger(Op, Lo, Hi); |
0 |
| 3611 |
return; |
0 |
3611 |
return; |
0 |
| 3612 |
} |
--- |
3612 |
} |
--- |
| 3613 |
|
--- |
3613 |
|
--- |
| 3614 |
RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT); |
0 |
3614 |
RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT); |
0 |
| 3615 |
assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!"); |
0 |
3615 |
assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!"); |
0 |
| 3616 |
TargetLowering::MakeLibCallOptions CallOptions; |
0 |
3616 |
TargetLowering::MakeLibCallOptions CallOptions; |
0 |
| 3617 |
CallOptions.setSExt(true); |
0 |
3617 |
CallOptions.setSExt(true); |
0 |
| 3618 |
std::pair Tmp = TLI.makeLibCall(DAG, LC, VT, Op, |
0 |
3618 |
std::pair Tmp = TLI.makeLibCall(DAG, LC, VT, Op, |
0 |
| 3619 |
CallOptions, dl, Chain); |
--- |
3619 |
CallOptions, dl, Chain); |
--- |
| 3620 |
SplitInteger(Tmp.first, Lo, Hi); |
0 |
3620 |
SplitInteger(Tmp.first, Lo, Hi); |
0 |
| 3621 |
|
--- |
3621 |
|
--- |
| 3622 |
if (IsStrict) |
0 |
3622 |
if (IsStrict) |
0 |
| 3623 |
ReplaceValueWith(SDValue(N, 1), Tmp.second); |
0 |
3623 |
ReplaceValueWith(SDValue(N, 1), Tmp.second); |
0 |
| 3624 |
} |
0 |
3624 |
} |
0 |
| 3625 |
|
--- |
3625 |
|
--- |
| 3626 |
void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo, |
0 |
3626 |
void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo, |
0 |
| 3627 |
SDValue &Hi) { |
--- |
3627 |
SDValue &Hi) { |
--- |
| 3628 |
SDLoc dl(N); |
0 |
3628 |
SDLoc dl(N); |
0 |
| 3629 |
EVT VT = N->getValueType(0); |
0 |
3629 |
EVT VT = N->getValueType(0); |
0 |
| 3630 |
|
--- |
3630 |
|
--- |
| 3631 |
bool IsStrict = N->isStrictFPOpcode(); |
0 |
3631 |
bool IsStrict = N->isStrictFPOpcode(); |
0 |
| 3632 |
SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); |
0 |
3632 |
SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); |
0 |
| 3633 |
SDValue Op = N->getOperand(IsStrict ? 1 : 0); |
0 |
3633 |
SDValue Op = N->getOperand(IsStrict ? 1 : 0); |
0 |
| 3634 |
if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) |
0 |
3634 |
if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) |
0 |
| 3635 |
Op = GetPromotedFloat(Op); |
0 |
3635 |
Op = GetPromotedFloat(Op); |
0 |
| 3636 |
|
--- |
3636 |
|
--- |
| 3637 |
if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) { |
0 |
3637 |
if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) { |
0 |
| 3638 |
EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()); |
0 |
3638 |
EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()); |
0 |
| 3639 |
Op = GetSoftPromotedHalf(Op); |
0 |
3639 |
Op = GetSoftPromotedHalf(Op); |
0 |
| 3640 |
Op = DAG.getNode(ISD::FP16_TO_FP, dl, NFPVT, Op); |
0 |
3640 |
Op = DAG.getNode(ISD::FP16_TO_FP, dl, NFPVT, Op); |
0 |
| 3641 |
Op = DAG.getNode(ISD::FP_TO_UINT, dl, VT, Op); |
0 |
3641 |
Op = DAG.getNode(ISD::FP_TO_UINT, dl, VT, Op); |
0 |
| 3642 |
SplitInteger(Op, Lo, Hi); |
0 |
3642 |
SplitInteger(Op, Lo, Hi); |
0 |
| 3643 |
return; |
0 |
3643 |
return; |
0 |
| 3644 |
} |
--- |
3644 |
} |
--- |
| 3645 |
|
--- |
3645 |
|
--- |
| 3646 |
RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT); |
0 |
3646 |
RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT); |
0 |
| 3647 |
assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!"); |
0 |
3647 |
assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!"); |
0 |
| 3648 |
TargetLowering::MakeLibCallOptions CallOptions; |
0 |
3648 |
TargetLowering::MakeLibCallOptions CallOptions; |
0 |
| 3649 |
std::pair Tmp = TLI.makeLibCall(DAG, LC, VT, Op, |
0 |
3649 |
std::pair Tmp = TLI.makeLibCall(DAG, LC, VT, Op, |
0 |
| 3650 |
CallOptions, dl, Chain); |
--- |
3650 |
CallOptions, dl, Chain); |
--- |
| 3651 |
SplitInteger(Tmp.first, Lo, Hi); |
0 |
3651 |
SplitInteger(Tmp.first, Lo, Hi); |
0 |
| 3652 |
|
--- |
3652 |
|
--- |
| 3653 |
if (IsStrict) |
0 |
3653 |
if (IsStrict) |
0 |
| 3654 |
ReplaceValueWith(SDValue(N, 1), Tmp.second); |
0 |
3654 |
ReplaceValueWith(SDValue(N, 1), Tmp.second); |
0 |
| 3655 |
} |
0 |
3655 |
} |
0 |
| 3656 |
|
--- |
3656 |
|
--- |
| 3657 |
void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo, |
0 |
3657 |
void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo, |
0 |
| 3658 |
SDValue &Hi) { |
--- |
3658 |
SDValue &Hi) { |
--- |
| 3659 |
SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG); |
0 |
3659 |
SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG); |
0 |
| 3660 |
SplitInteger(Res, Lo, Hi); |
0 |
3660 |
SplitInteger(Res, Lo, Hi); |
0 |
| 3661 |
} |
0 |
3661 |
} |
0 |
| 3662 |
|
--- |
3662 |
|
--- |
| 3663 |
void DAGTypeLegalizer::ExpandIntRes_XROUND_XRINT(SDNode *N, SDValue &Lo, |
0 |
3663 |
void DAGTypeLegalizer::ExpandIntRes_XROUND_XRINT(SDNode *N, SDValue &Lo, |
0 |
| 3664 |
SDValue &Hi) { |
--- |
3664 |
SDValue &Hi) { |
--- |
| 3665 |
SDLoc dl(N); |
0 |
3665 |
SDLoc dl(N); |
0 |
| 3666 |
bool IsStrict = N->isStrictFPOpcode(); |
0 |
3666 |
bool IsStrict = N->isStrictFPOpcode(); |
0 |
| 3667 |
SDValue Op = N->getOperand(IsStrict ? 1 : 0); |
0 |
3667 |
SDValue Op = N->getOperand(IsStrict ? 1 : 0); |
0 |
| 3668 |
SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); |
0 |
3668 |
SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); |
0 |
| 3669 |
|
--- |
3669 |
|
--- |
| 3670 |
assert(getTypeAction(Op.getValueType()) != TargetLowering::TypePromoteFloat && |
0 |
3670 |
assert(getTypeAction(Op.getValueType()) != TargetLowering::TypePromoteFloat && |
0 |
| 3671 |
"Input type needs to be promoted!"); |
--- |
3671 |
"Input type needs to be promoted!"); |
--- |
| 3672 |
|
--- |
3672 |
|
--- |
| 3673 |
EVT VT = Op.getValueType(); |
0 |
3673 |
EVT VT = Op.getValueType(); |
0 |
| 3674 |
|
--- |
3674 |
|
--- |
| 3675 |
if (VT == MVT::f16) { |
0 |
3675 |
if (VT == MVT::f16) { |
0 |
| 3676 |
VT = MVT::f32; |
0 |
3676 |
VT = MVT::f32; |
0 |
| 3677 |
// Extend to f32. |
--- |
3677 |
// Extend to f32. |
--- |
| 3678 |
if (IsStrict) { |
0 |
3678 |
if (IsStrict) { |
0 |
| 3679 |
Op = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { VT, MVT::Other }, {Chain, Op}); |
0 |
3679 |
Op = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { VT, MVT::Other }, {Chain, Op}); |
0 |
| 3680 |
Chain = Op.getValue(1); |
0 |
3680 |
Chain = Op.getValue(1); |
0 |
| 3681 |
} else { |
--- |
3681 |
} else { |
--- |
| 3682 |
Op = DAG.getNode(ISD::FP_EXTEND, dl, VT, Op); |
0 |
3682 |
Op = DAG.getNode(ISD::FP_EXTEND, dl, VT, Op); |
0 |
| 3683 |
} |
--- |
3683 |
} |
--- |
| 3684 |
} |
--- |
3684 |
} |
--- |
| 3685 |
|
--- |
3685 |
|
--- |
| 3686 |
RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
0 |
3686 |
RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
0 |
| 3687 |
if (N->getOpcode() == ISD::LROUND || |
0 |
3687 |
if (N->getOpcode() == ISD::LROUND || |
0 |
| 3688 |
N->getOpcode() == ISD::STRICT_LROUND) { |
0 |
3688 |
N->getOpcode() == ISD::STRICT_LROUND) { |
0 |
| 3689 |
if (VT == MVT::f32) |
0 |
3689 |
if (VT == MVT::f32) |
0 |
| 3690 |
LC = RTLIB::LROUND_F32; |
0 |
3690 |
LC = RTLIB::LROUND_F32; |
0 |
| 3691 |
else if (VT == MVT::f64) |
0 |
3691 |
else if (VT == MVT::f64) |
0 |
| 3692 |
LC = RTLIB::LROUND_F64; |
0 |
3692 |
LC = RTLIB::LROUND_F64; |
0 |
| 3693 |
else if (VT == MVT::f80) |
0 |
3693 |
else if (VT == MVT::f80) |
0 |
| 3694 |
LC = RTLIB::LROUND_F80; |
0 |
3694 |
LC = RTLIB::LROUND_F80; |
0 |
| 3695 |
else if (VT == MVT::f128) |
0 |
3695 |
else if (VT == MVT::f128) |
0 |
| 3696 |
LC = RTLIB::LROUND_F128; |
0 |
3696 |
LC = RTLIB::LROUND_F128; |
0 |
| 3697 |
else if (VT == MVT::ppcf128) |
0 |
3697 |
else if (VT == MVT::ppcf128) |
0 |
| 3698 |
LC = RTLIB::LROUND_PPCF128; |
0 |
3698 |
LC = RTLIB::LROUND_PPCF128; |
0 |
| 3699 |
assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected lround input type!"); |
0 |
3699 |
assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected lround input type!"); |
0 |
| 3700 |
} else if (N->getOpcode() == ISD::LRINT || |
0 |
3700 |
} else if (N->getOpcode() == ISD::LRINT || |
0 |
| 3701 |
N->getOpcode() == ISD::STRICT_LRINT) { |
0 |
3701 |
N->getOpcode() == ISD::STRICT_LRINT) { |
0 |
| 3702 |
if (VT == MVT::f32) |
0 |
3702 |
if (VT == MVT::f32) |
0 |
| 3703 |
LC = RTLIB::LRINT_F32; |
0 |
3703 |
LC = RTLIB::LRINT_F32; |
0 |
| 3704 |
else if (VT == MVT::f64) |
0 |
3704 |
else if (VT == MVT::f64) |
0 |
| 3705 |
LC = RTLIB::LRINT_F64; |
0 |
3705 |
LC = RTLIB::LRINT_F64; |
0 |
| 3706 |
else if (VT == MVT::f80) |
0 |
3706 |
else if (VT == MVT::f80) |
0 |
| 3707 |
LC = RTLIB::LRINT_F80; |
0 |
3707 |
LC = RTLIB::LRINT_F80; |
0 |
| 3708 |
else if (VT == MVT::f128) |
0 |
3708 |
else if (VT == MVT::f128) |
0 |
| 3709 |
LC = RTLIB::LRINT_F128; |
0 |
3709 |
LC = RTLIB::LRINT_F128; |
0 |
| 3710 |
else if (VT == MVT::ppcf128) |
0 |
3710 |
else if (VT == MVT::ppcf128) |
0 |
| 3711 |
LC = RTLIB::LRINT_PPCF128; |
0 |
3711 |
LC = RTLIB::LRINT_PPCF128; |
0 |
| 3712 |
assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected lrint input type!"); |
0 |
3712 |
assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected lrint input type!"); |
0 |
| 3713 |
} else if (N->getOpcode() == ISD::LLROUND || |
0 |
3713 |
} else if (N->getOpcode() == ISD::LLROUND || |
0 |
| 3714 |
N->getOpcode() == ISD::STRICT_LLROUND) { |
0 |
3714 |
N->getOpcode() == ISD::STRICT_LLROUND) { |
0 |
| 3715 |
if (VT == MVT::f32) |
0 |
3715 |
if (VT == MVT::f32) |
0 |
| 3716 |
LC = RTLIB::LLROUND_F32; |
0 |
3716 |
LC = RTLIB::LLROUND_F32; |
0 |
| 3717 |
else if (VT == MVT::f64) |
0 |
3717 |
else if (VT == MVT::f64) |
0 |
| 3718 |
LC = RTLIB::LLROUND_F64; |
0 |
3718 |
LC = RTLIB::LLROUND_F64; |
0 |
| 3719 |
else if (VT == MVT::f80) |
0 |
3719 |
else if (VT == MVT::f80) |
0 |
| 3720 |
LC = RTLIB::LLROUND_F80; |
0 |
3720 |
LC = RTLIB::LLROUND_F80; |
0 |
| 3721 |
else if (VT == MVT::f128) |
0 |
3721 |
else if (VT == MVT::f128) |
0 |
| 3722 |
LC = RTLIB::LLROUND_F128; |
0 |
3722 |
LC = RTLIB::LLROUND_F128; |
0 |
| 3723 |
else if (VT == MVT::ppcf128) |
0 |
3723 |
else if (VT == MVT::ppcf128) |
0 |
| 3724 |
LC = RTLIB::LLROUND_PPCF128; |
0 |
3724 |
LC = RTLIB::LLROUND_PPCF128; |
0 |
| 3725 |
assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llround input type!"); |
0 |
3725 |
assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llround input type!"); |
0 |
| 3726 |
} else if (N->getOpcode() == ISD::LLRINT || |
0 |
3726 |
} else if (N->getOpcode() == ISD::LLRINT || |
0 |
| 3727 |
N->getOpcode() == ISD::STRICT_LLRINT) { |
0 |
3727 |
N->getOpcode() == ISD::STRICT_LLRINT) { |
0 |
| 3728 |
if (VT == MVT::f32) |
0 |
3728 |
if (VT == MVT::f32) |
0 |
| 3729 |
LC = RTLIB::LLRINT_F32; |
0 |
3729 |
LC = RTLIB::LLRINT_F32; |
0 |
| 3730 |
else if (VT == MVT::f64) |
0 |
3730 |
else if (VT == MVT::f64) |
0 |
| 3731 |
LC = RTLIB::LLRINT_F64; |
0 |
3731 |
LC = RTLIB::LLRINT_F64; |
0 |
| 3732 |
else if (VT == MVT::f80) |
0 |
3732 |
else if (VT == MVT::f80) |
0 |
| 3733 |
LC = RTLIB::LLRINT_F80; |
0 |
3733 |
LC = RTLIB::LLRINT_F80; |
0 |
| 3734 |
else if (VT == MVT::f128) |
0 |
3734 |
else if (VT == MVT::f128) |
0 |
| 3735 |
LC = RTLIB::LLRINT_F128; |
0 |
3735 |
LC = RTLIB::LLRINT_F128; |
0 |
| 3736 |
else if (VT == MVT::ppcf128) |
0 |
3736 |
else if (VT == MVT::ppcf128) |
0 |
| 3737 |
LC = RTLIB::LLRINT_PPCF128; |
0 |
3737 |
LC = RTLIB::LLRINT_PPCF128; |
0 |
| 3738 |
assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llrint input type!"); |
0 |
3738 |
assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llrint input type!"); |
0 |
| 3739 |
} else |
--- |
3739 |
} else |
--- |
| 3740 |
llvm_unreachable("Unexpected opcode!"); |
0 |
3740 |
llvm_unreachable("Unexpected opcode!"); |
0 |
| 3741 |
|
--- |
3741 |
|
--- |
| 3742 |
EVT RetVT = N->getValueType(0); |
0 |
3742 |
EVT RetVT = N->getValueType(0); |
0 |
| 3743 |
|
--- |
3743 |
|
--- |
| 3744 |
TargetLowering::MakeLibCallOptions CallOptions; |
0 |
3744 |
TargetLowering::MakeLibCallOptions CallOptions; |
0 |
| 3745 |
CallOptions.setSExt(true); |
0 |
3745 |
CallOptions.setSExt(true); |
0 |
| 3746 |
std::pair Tmp = TLI.makeLibCall(DAG, LC, RetVT, |
0 |
3746 |
std::pair Tmp = TLI.makeLibCall(DAG, LC, RetVT, |
0 |
| 3747 |
Op, CallOptions, dl, |
--- |
3747 |
Op, CallOptions, dl, |
--- |
| 3748 |
Chain); |
--- |
3748 |
Chain); |
--- |
| 3749 |
SplitInteger(Tmp.first, Lo, Hi); |
0 |
3749 |
SplitInteger(Tmp.first, Lo, Hi); |
0 |
| 3750 |
|
--- |
3750 |
|
--- |
| 3751 |
if (N->isStrictFPOpcode()) |
0 |
3751 |
if (N->isStrictFPOpcode()) |
0 |
| 3752 |
ReplaceValueWith(SDValue(N, 1), Tmp.second); |
0 |
3752 |
ReplaceValueWith(SDValue(N, 1), Tmp.second); |
0 |
| 3753 |
} |
0 |
3753 |
} |
0 |
| 3754 |
|
--- |
3754 |
|
--- |
| 3755 |
void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N, |
0 |
3755 |
void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N, |
0 |
| 3756 |
SDValue &Lo, SDValue &Hi) { |
--- |
3756 |
SDValue &Lo, SDValue &Hi) { |
--- |
| 3757 |
if (N->isAtomic()) { |
0 |
3757 |
if (N->isAtomic()) { |
0 |
| 3758 |
// It's typical to have larger CAS than atomic load instructions. |
--- |
3758 |
// It's typical to have larger CAS than atomic load instructions. |
--- |
| 3759 |
SDLoc dl(N); |
0 |
3759 |
SDLoc dl(N); |
0 |
| 3760 |
EVT VT = N->getMemoryVT(); |
0 |
3760 |
EVT VT = N->getMemoryVT(); |
0 |
| 3761 |
SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other); |
0 |
3761 |
SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other); |
0 |
| 3762 |
SDValue Zero = DAG.getConstant(0, dl, VT); |
0 |
3762 |
SDValue Zero = DAG.getConstant(0, dl, VT); |
0 |
| 3763 |
SDValue Swap = DAG.getAtomicCmpSwap( |
0 |
3763 |
SDValue Swap = DAG.getAtomicCmpSwap( |
0 |
| 3764 |
ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, |
--- |
3764 |
ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, |
--- |
| 3765 |
VT, VTs, N->getOperand(0), |
0 |
3765 |
VT, VTs, N->getOperand(0), |
0 |
| 3766 |
N->getOperand(1), Zero, Zero, N->getMemOperand()); |
0 |
3766 |
N->getOperand(1), Zero, Zero, N->getMemOperand()); |
0 |
| 3767 |
ReplaceValueWith(SDValue(N, 0), Swap.getValue(0)); |
0 |
3767 |
ReplaceValueWith(SDValue(N, 0), Swap.getValue(0)); |
0 |
| 3768 |
ReplaceValueWith(SDValue(N, 1), Swap.getValue(2)); |
0 |
3768 |
ReplaceValueWith(SDValue(N, 1), Swap.getValue(2)); |
0 |
| 3769 |
return; |
0 |
3769 |
return; |
0 |
| 3770 |
} |
0 |
3770 |
} |
0 |
| 3771 |
|
--- |
3771 |
|
--- |
| 3772 |
if (ISD::isNormalLoad(N)) { |
0 |
3772 |
if (ISD::isNormalLoad(N)) { |
0 |
| 3773 |
ExpandRes_NormalLoad(N, Lo, Hi); |
0 |
3773 |
ExpandRes_NormalLoad(N, Lo, Hi); |
0 |
| 3774 |
return; |
0 |
3774 |
return; |
0 |
| 3775 |
} |
--- |
3775 |
} |
--- |
| 3776 |
|
--- |
3776 |
|
--- |
| 3777 |
assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); |
0 |
3777 |
assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); |
0 |
| 3778 |
|
--- |
3778 |
|
--- |
| 3779 |
EVT VT = N->getValueType(0); |
0 |
3779 |
EVT VT = N->getValueType(0); |
0 |
| 3780 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); |
0 |
3780 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); |
0 |
| 3781 |
SDValue Ch = N->getChain(); |
0 |
3781 |
SDValue Ch = N->getChain(); |
0 |
| 3782 |
SDValue Ptr = N->getBasePtr(); |
0 |
3782 |
SDValue Ptr = N->getBasePtr(); |
0 |
| 3783 |
ISD::LoadExtType ExtType = N->getExtensionType(); |
0 |
3783 |
ISD::LoadExtType ExtType = N->getExtensionType(); |
0 |
| 3784 |
MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags(); |
0 |
3784 |
MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags(); |
0 |
| 3785 |
AAMDNodes AAInfo = N->getAAInfo(); |
0 |
3785 |
AAMDNodes AAInfo = N->getAAInfo(); |
0 |
| 3786 |
SDLoc dl(N); |
0 |
3786 |
SDLoc dl(N); |
0 |
| 3787 |
|
--- |
3787 |
|
--- |
| 3788 |
assert(NVT.isByteSized() && "Expanded type not byte sized!"); |
0 |
3788 |
assert(NVT.isByteSized() && "Expanded type not byte sized!"); |
0 |
| 3789 |
|
--- |
3789 |
|
--- |
| 3790 |
if (N->getMemoryVT().bitsLE(NVT)) { |
0 |
3790 |
if (N->getMemoryVT().bitsLE(NVT)) { |
0 |
| 3791 |
EVT MemVT = N->getMemoryVT(); |
0 |
3791 |
EVT MemVT = N->getMemoryVT(); |
0 |
| 3792 |
|
--- |
3792 |
|
--- |
| 3793 |
Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT, |
0 |
3793 |
Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT, |
0 |
| 3794 |
N->getOriginalAlign(), MMOFlags, AAInfo); |
--- |
3794 |
N->getOriginalAlign(), MMOFlags, AAInfo); |
--- |
| 3795 |
|
--- |
3795 |
|
--- |
| 3796 |
// Remember the chain. |
--- |
3796 |
// Remember the chain. |
--- |
| 3797 |
Ch = Lo.getValue(1); |
0 |
3797 |
Ch = Lo.getValue(1); |
0 |
| 3798 |
|
--- |
3798 |
|
--- |
| 3799 |
if (ExtType == ISD::SEXTLOAD) { |
0 |
3799 |
if (ExtType == ISD::SEXTLOAD) { |
0 |
| 3800 |
// The high part is obtained by SRA'ing all but one of the bits of the |
--- |
3800 |
// The high part is obtained by SRA'ing all but one of the bits of the |
--- |
| 3801 |
// lo part. |
--- |
3801 |
// lo part. |
--- |
| 3802 |
unsigned LoSize = Lo.getValueSizeInBits(); |
0 |
3802 |
unsigned LoSize = Lo.getValueSizeInBits(); |
0 |
| 3803 |
Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, |
0 |
3803 |
Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, |
0 |
| 3804 |
DAG.getConstant(LoSize - 1, dl, |
0 |
3804 |
DAG.getConstant(LoSize - 1, dl, |
0 |
| 3805 |
TLI.getPointerTy(DAG.getDataLayout()))); |
0 |
3805 |
TLI.getPointerTy(DAG.getDataLayout()))); |
0 |
| 3806 |
} else if (ExtType == ISD::ZEXTLOAD) { |
0 |
3806 |
} else if (ExtType == ISD::ZEXTLOAD) { |
0 |
| 3807 |
// The high part is just a zero. |
--- |
3807 |
// The high part is just a zero. |
--- |
| 3808 |
Hi = DAG.getConstant(0, dl, NVT); |
0 |
3808 |
Hi = DAG.getConstant(0, dl, NVT); |
0 |
| 3809 |
} else { |
--- |
3809 |
} else { |
--- |
| 3810 |
assert(ExtType == ISD::EXTLOAD && "Unknown extload!"); |
0 |
3810 |
assert(ExtType == ISD::EXTLOAD && "Unknown extload!"); |
0 |
| 3811 |
// The high part is undefined. |
--- |
3811 |
// The high part is undefined. |
--- |
| 3812 |
Hi = DAG.getUNDEF(NVT); |
0 |
3812 |
Hi = DAG.getUNDEF(NVT); |
0 |
| 3813 |
} |
--- |
3813 |
} |
--- |
| 3814 |
} else if (DAG.getDataLayout().isLittleEndian()) { |
0 |
3814 |
} else if (DAG.getDataLayout().isLittleEndian()) { |
0 |
| 3815 |
// Little-endian - low bits are at low addresses. |
--- |
3815 |
// Little-endian - low bits are at low addresses. |
--- |
| 3816 |
Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), |
0 |
3816 |
Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), |
0 |
| 3817 |
N->getOriginalAlign(), MMOFlags, AAInfo); |
--- |
3817 |
N->getOriginalAlign(), MMOFlags, AAInfo); |
--- |
| 3818 |
|
--- |
3818 |
|
--- |
| 3819 |
unsigned ExcessBits = |
--- |
3819 |
unsigned ExcessBits = |
--- |
| 3820 |
N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); |
0 |
3820 |
N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); |
0 |
| 3821 |
EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); |
0 |
3821 |
EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); |
0 |
| 3822 |
|
--- |
3822 |
|
--- |
| 3823 |
// Increment the pointer to the other half. |
--- |
3823 |
// Increment the pointer to the other half. |
--- |
| 3824 |
unsigned IncrementSize = NVT.getSizeInBits()/8; |
0 |
3824 |
unsigned IncrementSize = NVT.getSizeInBits()/8; |
0 |
| 3825 |
Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl); |
0 |
3825 |
Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl); |
0 |
| 3826 |
Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, |
0 |
3826 |
Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, |
0 |
| 3827 |
N->getPointerInfo().getWithOffset(IncrementSize), NEVT, |
0 |
3827 |
N->getPointerInfo().getWithOffset(IncrementSize), NEVT, |
0 |
| 3828 |
N->getOriginalAlign(), MMOFlags, AAInfo); |
--- |
3828 |
N->getOriginalAlign(), MMOFlags, AAInfo); |
--- |
| 3829 |
|
--- |
3829 |
|
--- |
| 3830 |
// Build a factor node to remember that this load is independent of the |
--- |
3830 |
// Build a factor node to remember that this load is independent of the |
--- |
| 3831 |
// other one. |
--- |
3831 |
// other one. |
--- |
| 3832 |
Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), |
0 |
3832 |
Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), |
0 |
| 3833 |
Hi.getValue(1)); |
--- |
3833 |
Hi.getValue(1)); |
--- |
| 3834 |
} else { |
--- |
3834 |
} else { |
--- |
| 3835 |
// Big-endian - high bits are at low addresses. Favor aligned loads at |
--- |
3835 |
// Big-endian - high bits are at low addresses. Favor aligned loads at |
--- |
| 3836 |
// the cost of some bit-fiddling. |
--- |
3836 |
// the cost of some bit-fiddling. |
--- |
| 3837 |
EVT MemVT = N->getMemoryVT(); |
0 |
3837 |
EVT MemVT = N->getMemoryVT(); |
0 |
| 3838 |
unsigned EBytes = MemVT.getStoreSize(); |
0 |
3838 |
unsigned EBytes = MemVT.getStoreSize(); |
0 |
| 3839 |
unsigned IncrementSize = NVT.getSizeInBits()/8; |
0 |
3839 |
unsigned IncrementSize = NVT.getSizeInBits()/8; |
0 |
| 3840 |
unsigned ExcessBits = (EBytes - IncrementSize)*8; |
0 |
3840 |
unsigned ExcessBits = (EBytes - IncrementSize)*8; |
0 |
| 3841 |
|
--- |
3841 |
|
--- |
| 3842 |
// Load both the high bits and maybe some of the low bits. |
--- |
3842 |
// Load both the high bits and maybe some of the low bits. |
--- |
| 3843 |
Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), |
0 |
3843 |
Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), |
0 |
| 3844 |
EVT::getIntegerVT(*DAG.getContext(), |
0 |
3844 |
EVT::getIntegerVT(*DAG.getContext(), |
0 |
| 3845 |
MemVT.getSizeInBits() - ExcessBits), |
0 |
3845 |
MemVT.getSizeInBits() - ExcessBits), |
0 |
| 3846 |
N->getOriginalAlign(), MMOFlags, AAInfo); |
--- |
3846 |
N->getOriginalAlign(), MMOFlags, AAInfo); |
--- |
| 3847 |
|
--- |
3847 |
|
--- |
| 3848 |
// Increment the pointer to the other half. |
--- |
3848 |
// Increment the pointer to the other half. |
--- |
| 3849 |
Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl); |
0 |
3849 |
Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl); |
0 |
| 3850 |
// Load the rest of the low bits. |
--- |
3850 |
// Load the rest of the low bits. |
--- |
| 3851 |
Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr, |
0 |
3851 |
Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr, |
0 |
| 3852 |
N->getPointerInfo().getWithOffset(IncrementSize), |
0 |
3852 |
N->getPointerInfo().getWithOffset(IncrementSize), |
0 |
| 3853 |
EVT::getIntegerVT(*DAG.getContext(), ExcessBits), |
0 |
3853 |
EVT::getIntegerVT(*DAG.getContext(), ExcessBits), |
0 |
| 3854 |
N->getOriginalAlign(), MMOFlags, AAInfo); |
--- |
3854 |
N->getOriginalAlign(), MMOFlags, AAInfo); |
--- |
| 3855 |
|
--- |
3855 |
|
--- |
| 3856 |
// Build a factor node to remember that this load is independent of the |
--- |
3856 |
// Build a factor node to remember that this load is independent of the |
--- |
| 3857 |
// other one. |
--- |
3857 |
// other one. |
--- |
| 3858 |
Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), |
0 |
3858 |
Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), |
0 |
| 3859 |
Hi.getValue(1)); |
--- |
3859 |
Hi.getValue(1)); |
--- |
| 3860 |
|
--- |
3860 |
|
--- |
| 3861 |
if (ExcessBits < NVT.getSizeInBits()) { |
0 |
3861 |
if (ExcessBits < NVT.getSizeInBits()) { |
0 |
| 3862 |
// Transfer low bits from the bottom of Hi to the top of Lo. |
--- |
3862 |
// Transfer low bits from the bottom of Hi to the top of Lo. |
--- |
| 3863 |
Lo = DAG.getNode( |
0 |
3863 |
Lo = DAG.getNode( |
0 |
| 3864 |
ISD::OR, dl, NVT, Lo, |
--- |
3864 |
ISD::OR, dl, NVT, Lo, |
--- |
| 3865 |
DAG.getNode(ISD::SHL, dl, NVT, Hi, |
0 |
3865 |
DAG.getNode(ISD::SHL, dl, NVT, Hi, |
0 |
| 3866 |
DAG.getConstant(ExcessBits, dl, |
0 |
3866 |
DAG.getConstant(ExcessBits, dl, |
0 |
| 3867 |
TLI.getPointerTy(DAG.getDataLayout())))); |
0 |
3867 |
TLI.getPointerTy(DAG.getDataLayout())))); |
0 |
| 3868 |
// Move high bits to the right position in Hi. |
--- |
3868 |
// Move high bits to the right position in Hi. |
--- |
| 3869 |
Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT, |
0 |
3869 |
Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT, |
0 |
| 3870 |
Hi, |
--- |
3870 |
Hi, |
--- |
| 3871 |
DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl, |
0 |
3871 |
DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl, |
0 |
| 3872 |
TLI.getPointerTy(DAG.getDataLayout()))); |
0 |
3872 |
TLI.getPointerTy(DAG.getDataLayout()))); |
0 |
| 3873 |
} |
--- |
3873 |
} |
--- |
| 3874 |
} |
--- |
3874 |
} |
--- |
| 3875 |
|
--- |
3875 |
|
--- |
| 3876 |
// Legalize the chain result - switch anything that used the old chain to |
--- |
3876 |
// Legalize the chain result - switch anything that used the old chain to |
--- |
| 3877 |
// use the new one. |
--- |
3877 |
// use the new one. |
--- |
| 3878 |
ReplaceValueWith(SDValue(N, 1), Ch); |
0 |
3878 |
ReplaceValueWith(SDValue(N, 1), Ch); |
0 |
| 3879 |
} |
0 |
3879 |
} |
0 |
| 3880 |
|
--- |
3880 |
|
--- |
| 3881 |
void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N, |
0 |
3881 |
void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N, |
0 |
| 3882 |
SDValue &Lo, SDValue &Hi) { |
--- |
3882 |
SDValue &Lo, SDValue &Hi) { |
--- |
| 3883 |
SDLoc dl(N); |
0 |
3883 |
SDLoc dl(N); |
0 |
| 3884 |
SDValue LL, LH, RL, RH; |
0 |
3884 |
SDValue LL, LH, RL, RH; |
0 |
| 3885 |
GetExpandedInteger(N->getOperand(0), LL, LH); |
0 |
3885 |
GetExpandedInteger(N->getOperand(0), LL, LH); |
0 |
| 3886 |
GetExpandedInteger(N->getOperand(1), RL, RH); |
0 |
3886 |
GetExpandedInteger(N->getOperand(1), RL, RH); |
0 |
| 3887 |
Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL); |
0 |
3887 |
Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL); |
0 |
| 3888 |
Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH); |
0 |
3888 |
Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH); |
0 |
| 3889 |
} |
0 |
3889 |
} |
0 |
| 3890 |
|
--- |
3890 |
|
--- |
| 3891 |
void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N, |
0 |
3891 |
void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N, |
0 |
| 3892 |
SDValue &Lo, SDValue &Hi) { |
--- |
3892 |
SDValue &Lo, SDValue &Hi) { |
--- |
| 3893 |
EVT VT = N->getValueType(0); |
0 |
3893 |
EVT VT = N->getValueType(0); |
0 |
| 3894 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); |
0 |
3894 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); |
0 |
| 3895 |
SDLoc dl(N); |
0 |
3895 |
SDLoc dl(N); |
0 |
| 3896 |
|
--- |
3896 |
|
--- |
| 3897 |
SDValue LL, LH, RL, RH; |
0 |
3897 |
SDValue LL, LH, RL, RH; |
0 |
| 3898 |
GetExpandedInteger(N->getOperand(0), LL, LH); |
0 |
3898 |
GetExpandedInteger(N->getOperand(0), LL, LH); |
0 |
| 3899 |
GetExpandedInteger(N->getOperand(1), RL, RH); |
0 |
3899 |
GetExpandedInteger(N->getOperand(1), RL, RH); |
0 |
| 3900 |
|
--- |
3900 |
|
--- |
| 3901 |
if (TLI.expandMUL(N, Lo, Hi, NVT, DAG, |
0 |
3901 |
if (TLI.expandMUL(N, Lo, Hi, NVT, DAG, |
0 |
| 3902 |
TargetLowering::MulExpansionKind::OnlyLegalOrCustom, |
--- |
3902 |
TargetLowering::MulExpansionKind::OnlyLegalOrCustom, |
--- |
| 3903 |
LL, LH, RL, RH)) |
--- |
3903 |
LL, LH, RL, RH)) |
--- |
| 3904 |
return; |
0 |
3904 |
return; |
0 |
| 3905 |
|
--- |
3905 |
|
--- |
| 3906 |
// If nothing else, we can make a libcall. |
--- |
3906 |
// If nothing else, we can make a libcall. |
--- |
| 3907 |
RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
0 |
3907 |
RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
0 |
| 3908 |
if (VT == MVT::i16) |
0 |
3908 |
if (VT == MVT::i16) |
0 |
| 3909 |
LC = RTLIB::MUL_I16; |
0 |
3909 |
LC = RTLIB::MUL_I16; |
0 |
| 3910 |
else if (VT == MVT::i32) |
0 |
3910 |
else if (VT == MVT::i32) |
0 |
| 3911 |
LC = RTLIB::MUL_I32; |
0 |
3911 |
LC = RTLIB::MUL_I32; |
0 |
| 3912 |
else if (VT == MVT::i64) |
0 |
3912 |
else if (VT == MVT::i64) |
0 |
| 3913 |
LC = RTLIB::MUL_I64; |
0 |
3913 |
LC = RTLIB::MUL_I64; |
0 |
| 3914 |
else if (VT == MVT::i128) |
0 |
3914 |
else if (VT == MVT::i128) |
0 |
| 3915 |
LC = RTLIB::MUL_I128; |
0 |
3915 |
LC = RTLIB::MUL_I128; |
0 |
| 3916 |
|
--- |
3916 |
|
--- |
| 3917 |
if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) { |
0 |
3917 |
if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) { |
0 |
| 3918 |
// We'll expand the multiplication by brute force because we have no other |
--- |
3918 |
// We'll expand the multiplication by brute force because we have no other |
--- |
| 3919 |
// options. This is a trivially-generalized version of the code from |
--- |
3919 |
// options. This is a trivially-generalized version of the code from |
--- |
| 3920 |
// Hacker's Delight (itself derived from Knuth's Algorithm M from section |
--- |
3920 |
// Hacker's Delight (itself derived from Knuth's Algorithm M from section |
--- |
| 3921 |
// 4.3.1). |
--- |
3921 |
// 4.3.1). |
--- |
| 3922 |
unsigned Bits = NVT.getSizeInBits(); |
0 |
3922 |
unsigned Bits = NVT.getSizeInBits(); |
0 |
| 3923 |
unsigned HalfBits = Bits >> 1; |
0 |
3923 |
unsigned HalfBits = Bits >> 1; |
0 |
| 3924 |
SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl, |
0 |
3924 |
SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl, |
0 |
| 3925 |
NVT); |
--- |
3925 |
NVT); |
--- |
| 3926 |
SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask); |
0 |
3926 |
SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask); |
0 |
| 3927 |
SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask); |
0 |
3927 |
SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask); |
0 |
| 3928 |
|
--- |
3928 |
|
--- |
| 3929 |
SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL); |
0 |
3929 |
SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL); |
0 |
| 3930 |
SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask); |
0 |
3930 |
SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask); |
0 |
| 3931 |
|
--- |
3931 |
|
--- |
| 3932 |
SDValue Shift = DAG.getShiftAmountConstant(HalfBits, NVT, dl); |
0 |
3932 |
SDValue Shift = DAG.getShiftAmountConstant(HalfBits, NVT, dl); |
0 |
| 3933 |
SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift); |
0 |
3933 |
SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift); |
0 |
| 3934 |
SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift); |
0 |
3934 |
SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift); |
0 |
| 3935 |
SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift); |
0 |
3935 |
SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift); |
0 |
| 3936 |
|
--- |
3936 |
|
--- |
| 3937 |
SDValue U = DAG.getNode(ISD::ADD, dl, NVT, |
0 |
3937 |
SDValue U = DAG.getNode(ISD::ADD, dl, NVT, |
0 |
| 3938 |
DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH); |
0 |
3938 |
DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH); |
0 |
| 3939 |
SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask); |
0 |
3939 |
SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask); |
0 |
| 3940 |
SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift); |
0 |
3940 |
SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift); |
0 |
| 3941 |
|
--- |
3941 |
|
--- |
| 3942 |
SDValue V = DAG.getNode(ISD::ADD, dl, NVT, |
0 |
3942 |
SDValue V = DAG.getNode(ISD::ADD, dl, NVT, |
0 |
| 3943 |
DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL); |
0 |
3943 |
DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL); |
0 |
| 3944 |
SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift); |
0 |
3944 |
SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift); |
0 |
| 3945 |
|
--- |
3945 |
|
--- |
| 3946 |
SDValue W = DAG.getNode(ISD::ADD, dl, NVT, |
0 |
3946 |
SDValue W = DAG.getNode(ISD::ADD, dl, NVT, |
0 |
| 3947 |
DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH), |
0 |
3947 |
DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH), |
0 |
| 3948 |
DAG.getNode(ISD::ADD, dl, NVT, UH, VH)); |
0 |
3948 |
DAG.getNode(ISD::ADD, dl, NVT, UH, VH)); |
0 |
| 3949 |
Lo = DAG.getNode(ISD::ADD, dl, NVT, TL, |
0 |
3949 |
Lo = DAG.getNode(ISD::ADD, dl, NVT, TL, |
0 |
| 3950 |
DAG.getNode(ISD::SHL, dl, NVT, V, Shift)); |
0 |
3950 |
DAG.getNode(ISD::SHL, dl, NVT, V, Shift)); |
0 |
| 3951 |
|
--- |
3951 |
|
--- |
| 3952 |
Hi = DAG.getNode(ISD::ADD, dl, NVT, W, |
0 |
3952 |
Hi = DAG.getNode(ISD::ADD, dl, NVT, W, |
0 |
| 3953 |
DAG.getNode(ISD::ADD, dl, NVT, |
0 |
3953 |
DAG.getNode(ISD::ADD, dl, NVT, |
0 |
| 3954 |
DAG.getNode(ISD::MUL, dl, NVT, RH, LL), |
0 |
3954 |
DAG.getNode(ISD::MUL, dl, NVT, RH, LL), |
0 |
| 3955 |
DAG.getNode(ISD::MUL, dl, NVT, RL, LH))); |
0 |
3955 |
DAG.getNode(ISD::MUL, dl, NVT, RL, LH))); |
0 |
| 3956 |
return; |
0 |
3956 |
return; |
0 |
| 3957 |
} |
--- |
3957 |
} |
--- |
| 3958 |
|
--- |
3958 |
|
--- |
| 3959 |
SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; |
0 |
3959 |
SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; |
0 |
| 3960 |
TargetLowering::MakeLibCallOptions CallOptions; |
0 |
3960 |
TargetLowering::MakeLibCallOptions CallOptions; |
0 |
| 3961 |
CallOptions.setSExt(true); |
0 |
3961 |
CallOptions.setSExt(true); |
0 |
| 3962 |
SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, |
0 |
3962 |
SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, |
0 |
| 3963 |
Lo, Hi); |
--- |
3963 |
Lo, Hi); |
--- |
| 3964 |
} |
0 |
3964 |
} |
0 |
| 3965 |
|
--- |
3965 |
|
--- |
| 3966 |
void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo, |
0 |
3966 |
void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo, |
0 |
| 3967 |
SDValue &Hi) { |
--- |
3967 |
SDValue &Hi) { |
--- |
| 3968 |
SDLoc DL(N); |
0 |
3968 |
SDLoc DL(N); |
0 |
| 3969 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
3969 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
| 3970 |
SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other); |
0 |
3970 |
SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other); |
0 |
| 3971 |
SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0)); |
0 |
3971 |
SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0)); |
0 |
| 3972 |
Lo = R.getValue(0); |
0 |
3972 |
Lo = R.getValue(0); |
0 |
| 3973 |
Hi = R.getValue(1); |
0 |
3973 |
Hi = R.getValue(1); |
0 |
| 3974 |
ReplaceValueWith(SDValue(N, 1), R.getValue(2)); |
0 |
3974 |
ReplaceValueWith(SDValue(N, 1), R.getValue(2)); |
0 |
| 3975 |
} |
0 |
3975 |
} |
0 |
| 3976 |
|
--- |
3976 |
|
--- |
| 3977 |
void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo, |
0 |
3977 |
void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo, |
0 |
| 3978 |
SDValue &Hi) { |
--- |
3978 |
SDValue &Hi) { |
--- |
| 3979 |
SDValue Result = TLI.expandAddSubSat(N, DAG); |
0 |
3979 |
SDValue Result = TLI.expandAddSubSat(N, DAG); |
0 |
| 3980 |
SplitInteger(Result, Lo, Hi); |
0 |
3980 |
SplitInteger(Result, Lo, Hi); |
0 |
| 3981 |
} |
0 |
3981 |
} |
0 |
| 3982 |
|
--- |
3982 |
|
--- |
| 3983 |
void DAGTypeLegalizer::ExpandIntRes_SHLSAT(SDNode *N, SDValue &Lo, |
0 |
3983 |
void DAGTypeLegalizer::ExpandIntRes_SHLSAT(SDNode *N, SDValue &Lo, |
0 |
| 3984 |
SDValue &Hi) { |
--- |
3984 |
SDValue &Hi) { |
--- |
| 3985 |
SDValue Result = TLI.expandShlSat(N, DAG); |
0 |
3985 |
SDValue Result = TLI.expandShlSat(N, DAG); |
0 |
| 3986 |
SplitInteger(Result, Lo, Hi); |
0 |
3986 |
SplitInteger(Result, Lo, Hi); |
0 |
| 3987 |
} |
0 |
3987 |
} |
0 |
| 3988 |
|
--- |
3988 |
|
--- |
| 3989 |
/// This performs an expansion of the integer result for a fixed point |
--- |
3989 |
/// This performs an expansion of the integer result for a fixed point |
--- |
| 3990 |
/// multiplication. The default expansion performs rounding down towards |
--- |
3990 |
/// multiplication. The default expansion performs rounding down towards |
--- |
| 3991 |
/// negative infinity, though targets that do care about rounding should specify |
--- |
3991 |
/// negative infinity, though targets that do care about rounding should specify |
--- |
| 3992 |
/// a target hook for rounding and provide their own expansion or lowering of |
--- |
3992 |
/// a target hook for rounding and provide their own expansion or lowering of |
--- |
| 3993 |
/// fixed point multiplication to be consistent with rounding. |
--- |
3993 |
/// fixed point multiplication to be consistent with rounding. |
--- |
| 3994 |
void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo, |
0 |
3994 |
void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo, |
0 |
| 3995 |
SDValue &Hi) { |
--- |
3995 |
SDValue &Hi) { |
--- |
| 3996 |
SDLoc dl(N); |
0 |
3996 |
SDLoc dl(N); |
0 |
| 3997 |
EVT VT = N->getValueType(0); |
0 |
3997 |
EVT VT = N->getValueType(0); |
0 |
| 3998 |
unsigned VTSize = VT.getScalarSizeInBits(); |
0 |
3998 |
unsigned VTSize = VT.getScalarSizeInBits(); |
0 |
| 3999 |
SDValue LHS = N->getOperand(0); |
0 |
3999 |
SDValue LHS = N->getOperand(0); |
0 |
| 4000 |
SDValue RHS = N->getOperand(1); |
0 |
4000 |
SDValue RHS = N->getOperand(1); |
0 |
| 4001 |
uint64_t Scale = N->getConstantOperandVal(2); |
0 |
4001 |
uint64_t Scale = N->getConstantOperandVal(2); |
0 |
| 4002 |
bool Saturating = (N->getOpcode() == ISD::SMULFIXSAT || |
0 |
4002 |
bool Saturating = (N->getOpcode() == ISD::SMULFIXSAT || |
0 |
| 4003 |
N->getOpcode() == ISD::UMULFIXSAT); |
0 |
4003 |
N->getOpcode() == ISD::UMULFIXSAT); |
0 |
| 4004 |
bool Signed = (N->getOpcode() == ISD::SMULFIX || |
0 |
4004 |
bool Signed = (N->getOpcode() == ISD::SMULFIX || |
0 |
| 4005 |
N->getOpcode() == ISD::SMULFIXSAT); |
0 |
4005 |
N->getOpcode() == ISD::SMULFIXSAT); |
0 |
| 4006 |
|
--- |
4006 |
|
--- |
| 4007 |
// Handle special case when scale is equal to zero. |
--- |
4007 |
// Handle special case when scale is equal to zero. |
--- |
| 4008 |
if (!Scale) { |
0 |
4008 |
if (!Scale) { |
0 |
| 4009 |
SDValue Result; |
0 |
4009 |
SDValue Result; |
0 |
| 4010 |
if (!Saturating) { |
0 |
4010 |
if (!Saturating) { |
0 |
| 4011 |
Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); |
0 |
4011 |
Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); |
0 |
| 4012 |
} else { |
--- |
4012 |
} else { |
--- |
| 4013 |
EVT BoolVT = getSetCCResultType(VT); |
0 |
4013 |
EVT BoolVT = getSetCCResultType(VT); |
0 |
| 4014 |
unsigned MulOp = Signed ? ISD::SMULO : ISD::UMULO; |
0 |
4014 |
unsigned MulOp = Signed ? ISD::SMULO : ISD::UMULO; |
0 |
| 4015 |
Result = DAG.getNode(MulOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS); |
0 |
4015 |
Result = DAG.getNode(MulOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS); |
0 |
| 4016 |
SDValue Product = Result.getValue(0); |
0 |
4016 |
SDValue Product = Result.getValue(0); |
0 |
| 4017 |
SDValue Overflow = Result.getValue(1); |
0 |
4017 |
SDValue Overflow = Result.getValue(1); |
0 |
| 4018 |
if (Signed) { |
0 |
4018 |
if (Signed) { |
0 |
| 4019 |
APInt MinVal = APInt::getSignedMinValue(VTSize); |
0 |
4019 |
APInt MinVal = APInt::getSignedMinValue(VTSize); |
0 |
| 4020 |
APInt MaxVal = APInt::getSignedMaxValue(VTSize); |
0 |
4020 |
APInt MaxVal = APInt::getSignedMaxValue(VTSize); |
0 |
| 4021 |
SDValue SatMin = DAG.getConstant(MinVal, dl, VT); |
0 |
4021 |
SDValue SatMin = DAG.getConstant(MinVal, dl, VT); |
0 |
| 4022 |
SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); |
0 |
4022 |
SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); |
0 |
| 4023 |
SDValue Zero = DAG.getConstant(0, dl, VT); |
0 |
4023 |
SDValue Zero = DAG.getConstant(0, dl, VT); |
0 |
| 4024 |
// Xor the inputs, if resulting sign bit is 0 the product will be |
--- |
4024 |
// Xor the inputs, if resulting sign bit is 0 the product will be |
--- |
| 4025 |
// positive, else negative. |
--- |
4025 |
// positive, else negative. |
--- |
| 4026 |
SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS); |
0 |
4026 |
SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS); |
0 |
| 4027 |
SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Xor, Zero, ISD::SETLT); |
0 |
4027 |
SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Xor, Zero, ISD::SETLT); |
0 |
| 4028 |
Result = DAG.getSelect(dl, VT, ProdNeg, SatMin, SatMax); |
0 |
4028 |
Result = DAG.getSelect(dl, VT, ProdNeg, SatMin, SatMax); |
0 |
| 4029 |
Result = DAG.getSelect(dl, VT, Overflow, Result, Product); |
0 |
4029 |
Result = DAG.getSelect(dl, VT, Overflow, Result, Product); |
0 |
| 4030 |
} else { |
0 |
4030 |
} else { |
0 |
| 4031 |
// For unsigned multiplication, we only need to check the max since we |
--- |
4031 |
// For unsigned multiplication, we only need to check the max since we |
--- |
| 4032 |
// can't really overflow towards zero. |
--- |
4032 |
// can't really overflow towards zero. |
--- |
| 4033 |
APInt MaxVal = APInt::getMaxValue(VTSize); |
0 |
4033 |
APInt MaxVal = APInt::getMaxValue(VTSize); |
0 |
| 4034 |
SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); |
0 |
4034 |
SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); |
0 |
| 4035 |
Result = DAG.getSelect(dl, VT, Overflow, SatMax, Product); |
0 |
4035 |
Result = DAG.getSelect(dl, VT, Overflow, SatMax, Product); |
0 |
| 4036 |
} |
0 |
4036 |
} |
0 |
| 4037 |
} |
--- |
4037 |
} |
--- |
| 4038 |
SplitInteger(Result, Lo, Hi); |
0 |
4038 |
SplitInteger(Result, Lo, Hi); |
0 |
| 4039 |
return; |
0 |
4039 |
return; |
0 |
| 4040 |
} |
--- |
4040 |
} |
--- |
| 4041 |
|
--- |
4041 |
|
--- |
| 4042 |
// For SMULFIX[SAT] we only expect to find Scale
| --- |
4042 |
// For SMULFIX[SAT] we only expect to find Scale
| --- |
| |
| 4043 |
// cover for unhandled cases below, while still being valid for UMULFIX[SAT]. |
--- |
4043 |
// cover for unhandled cases below, while still being valid for UMULFIX[SAT]. |
--- |
| 4044 |
assert(Scale <= VTSize && "Scale can't be larger than the value type size."); |
0 |
4044 |
assert(Scale <= VTSize && "Scale can't be larger than the value type size."); |
0 |
| 4045 |
|
--- |
4045 |
|
--- |
| 4046 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); |
0 |
4046 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); |
0 |
| 4047 |
SDValue LL, LH, RL, RH; |
0 |
4047 |
SDValue LL, LH, RL, RH; |
0 |
| 4048 |
GetExpandedInteger(LHS, LL, LH); |
0 |
4048 |
GetExpandedInteger(LHS, LL, LH); |
0 |
| 4049 |
GetExpandedInteger(RHS, RL, RH); |
0 |
4049 |
GetExpandedInteger(RHS, RL, RH); |
0 |
| 4050 |
SmallVector Result; |
0 |
4050 |
SmallVector Result; |
0 |
| 4051 |
|
--- |
4051 |
|
--- |
| 4052 |
unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI; |
0 |
4052 |
unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI; |
0 |
| 4053 |
if (!TLI.expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG, |
0 |
4053 |
if (!TLI.expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG, |
0 |
| 4054 |
TargetLowering::MulExpansionKind::OnlyLegalOrCustom, |
--- |
4054 |
TargetLowering::MulExpansionKind::OnlyLegalOrCustom, |
--- |
| 4055 |
LL, LH, RL, RH)) { |
--- |
4055 |
LL, LH, RL, RH)) { |
--- |
| 4056 |
report_fatal_error("Unable to expand MUL_FIX using MUL_LOHI."); |
0 |
4056 |
report_fatal_error("Unable to expand MUL_FIX using MUL_LOHI."); |
0 |
| 4057 |
return; |
--- |
4057 |
return; |
--- |
| 4058 |
} |
--- |
4058 |
} |
--- |
| 4059 |
|
--- |
4059 |
|
--- |
| 4060 |
unsigned NVTSize = NVT.getScalarSizeInBits(); |
0 |
4060 |
unsigned NVTSize = NVT.getScalarSizeInBits(); |
0 |
| 4061 |
assert((VTSize == NVTSize * 2) && "Expected the new value type to be half " |
0 |
4061 |
assert((VTSize == NVTSize * 2) && "Expected the new value type to be half " |
0 |
| 4062 |
"the size of the current value type"); |
--- |
4062 |
"the size of the current value type"); |
--- |
| 4063 |
|
--- |
4063 |
|
--- |
| 4064 |
// After getting the multiplication result in 4 parts, we need to perform a |
--- |
4064 |
// After getting the multiplication result in 4 parts, we need to perform a |
--- |
| 4065 |
// shift right by the amount of the scale to get the result in that scale. |
--- |
4065 |
// shift right by the amount of the scale to get the result in that scale. |
--- |
| 4066 |
// |
--- |
4066 |
// |
--- |
| 4067 |
// Let's say we multiply 2 64 bit numbers. The resulting value can be held in |
--- |
4067 |
// Let's say we multiply 2 64 bit numbers. The resulting value can be held in |
--- |
| 4068 |
// 128 bits that are cut into 4 32-bit parts: |
--- |
4068 |
// 128 bits that are cut into 4 32-bit parts: |
--- |
| 4069 |
// |
--- |
4069 |
// |
--- |
| 4070 |
// HH HL LH LL |
--- |
4070 |
// HH HL LH LL |
--- |
| 4071 |
// |---32---|---32---|---32---|---32---| |
--- |
4071 |
// |---32---|---32---|---32---|---32---| |
--- |
| 4072 |
// 128 96 64 32 0 |
--- |
4072 |
// 128 96 64 32 0 |
--- |
| 4073 |
// |
--- |
4073 |
// |
--- |
| 4074 |
// |------VTSize-----| |
--- |
4074 |
// |------VTSize-----| |
--- |
| 4075 |
// |
--- |
4075 |
// |
--- |
| 4076 |
// |NVTSize-| |
--- |
4076 |
// |NVTSize-| |
--- |
| 4077 |
// |
--- |
4077 |
// |
--- |
| 4078 |
// The resulting Lo and Hi would normally be in LL and LH after the shift. But |
--- |
4078 |
// The resulting Lo and Hi would normally be in LL and LH after the shift. But |
--- |
| 4079 |
// to avoid unneccessary shifting of all 4 parts, we can adjust the shift |
--- |
4079 |
// to avoid unneccessary shifting of all 4 parts, we can adjust the shift |
--- |
| 4080 |
// amount and get Lo and Hi using two funnel shifts. Or for the special case |
--- |
4080 |
// amount and get Lo and Hi using two funnel shifts. Or for the special case |
--- |
| 4081 |
// when Scale is a multiple of NVTSize we can just pick the result without |
--- |
4081 |
// when Scale is a multiple of NVTSize we can just pick the result without |
--- |
| 4082 |
// shifting. |
--- |
4082 |
// shifting. |
--- |
| 4083 |
uint64_t Part0 = Scale / NVTSize; // Part holding lowest bit needed. |
0 |
4083 |
uint64_t Part0 = Scale / NVTSize; // Part holding lowest bit needed. |
0 |
| 4084 |
if (Scale % NVTSize) { |
0 |
4084 |
if (Scale % NVTSize) { |
0 |
| 4085 |
SDValue ShiftAmount = DAG.getShiftAmountConstant(Scale % NVTSize, NVT, dl); |
0 |
4085 |
SDValue ShiftAmount = DAG.getShiftAmountConstant(Scale % NVTSize, NVT, dl); |
0 |
| 4086 |
Lo = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 1], Result[Part0], |
0 |
4086 |
Lo = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 1], Result[Part0], |
0 |
| 4087 |
ShiftAmount); |
--- |
4087 |
ShiftAmount); |
--- |
| 4088 |
Hi = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 2], Result[Part0 + 1], |
0 |
4088 |
Hi = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 2], Result[Part0 + 1], |
0 |
| 4089 |
ShiftAmount); |
--- |
4089 |
ShiftAmount); |
--- |
| 4090 |
} else { |
--- |
4090 |
} else { |
--- |
| 4091 |
Lo = Result[Part0]; |
0 |
4091 |
Lo = Result[Part0]; |
0 |
| 4092 |
Hi = Result[Part0 + 1]; |
0 |
4092 |
Hi = Result[Part0 + 1]; |
0 |
| 4093 |
} |
--- |
4093 |
} |
--- |
| 4094 |
|
--- |
4094 |
|
--- |
| 4095 |
// Unless saturation is requested we are done. The result is in . |
--- |
4095 |
// Unless saturation is requested we are done. The result is in . |
--- |
| 4096 |
if (!Saturating) |
0 |
4096 |
if (!Saturating) |
0 |
| 4097 |
return; |
0 |
4097 |
return; |
0 |
| 4098 |
|
--- |
4098 |
|
--- |
| 4099 |
// Can not overflow when there is no integer part. |
--- |
4099 |
// Can not overflow when there is no integer part. |
--- |
| 4100 |
if (Scale == VTSize) |
0 |
4100 |
if (Scale == VTSize) |
0 |
| 4101 |
return; |
0 |
4101 |
return; |
0 |
| 4102 |
|
--- |
4102 |
|
--- |
| 4103 |
// To handle saturation we must check for overflow in the multiplication. |
--- |
4103 |
// To handle saturation we must check for overflow in the multiplication. |
--- |
| 4104 |
// |
--- |
4104 |
// |
--- |
| 4105 |
// Unsigned overflow happened if the upper (VTSize - Scale) bits (of Result) |
--- |
4105 |
// Unsigned overflow happened if the upper (VTSize - Scale) bits (of Result) |
--- |
| 4106 |
// aren't all zeroes. |
--- |
4106 |
// aren't all zeroes. |
--- |
| 4107 |
// |
--- |
4107 |
// |
--- |
| 4108 |
// Signed overflow happened if the upper (VTSize - Scale + 1) bits (of Result) |
--- |
4108 |
// Signed overflow happened if the upper (VTSize - Scale + 1) bits (of Result) |
--- |
| 4109 |
// aren't all ones or all zeroes. |
--- |
4109 |
// aren't all ones or all zeroes. |
--- |
| 4110 |
// |
--- |
4110 |
// |
--- |
| 4111 |
// We cannot overflow past HH when multiplying 2 ints of size VTSize, so the |
--- |
4111 |
// We cannot overflow past HH when multiplying 2 ints of size VTSize, so the |
--- |
| 4112 |
// highest bit of HH determines saturation direction in the event of signed |
--- |
4112 |
// highest bit of HH determines saturation direction in the event of signed |
--- |
| 4113 |
// saturation. |
--- |
4113 |
// saturation. |
--- |
| 4114 |
|
--- |
4114 |
|
--- |
| 4115 |
SDValue ResultHL = Result[2]; |
0 |
4115 |
SDValue ResultHL = Result[2]; |
0 |
| 4116 |
SDValue ResultHH = Result[3]; |
0 |
4116 |
SDValue ResultHH = Result[3]; |
0 |
| 4117 |
|
--- |
4117 |
|
--- |
| 4118 |
SDValue SatMax, SatMin; |
0 |
4118 |
SDValue SatMax, SatMin; |
0 |
| 4119 |
SDValue NVTZero = DAG.getConstant(0, dl, NVT); |
0 |
4119 |
SDValue NVTZero = DAG.getConstant(0, dl, NVT); |
0 |
| 4120 |
SDValue NVTNeg1 = DAG.getConstant(-1, dl, NVT); |
0 |
4120 |
SDValue NVTNeg1 = DAG.getConstant(-1, dl, NVT); |
0 |
| 4121 |
EVT BoolNVT = getSetCCResultType(NVT); |
0 |
4121 |
EVT BoolNVT = getSetCCResultType(NVT); |
0 |
| 4122 |
|
--- |
4122 |
|
--- |
| 4123 |
if (!Signed) { |
0 |
4123 |
if (!Signed) { |
0 |
| 4124 |
if (Scale < NVTSize) { |
0 |
4124 |
if (Scale < NVTSize) { |
0 |
| 4125 |
// Overflow happened if ((HH | (HL >> Scale)) != 0). |
--- |
4125 |
// Overflow happened if ((HH | (HL >> Scale)) != 0). |
--- |
| 4126 |
SDValue HLAdjusted = |
--- |
4126 |
SDValue HLAdjusted = |
--- |
| 4127 |
DAG.getNode(ISD::SRL, dl, NVT, ResultHL, |
0 |
4127 |
DAG.getNode(ISD::SRL, dl, NVT, ResultHL, |
0 |
| 4128 |
DAG.getShiftAmountConstant(Scale, NVT, dl)); |
0 |
4128 |
DAG.getShiftAmountConstant(Scale, NVT, dl)); |
0 |
| 4129 |
SDValue Tmp = DAG.getNode(ISD::OR, dl, NVT, HLAdjusted, ResultHH); |
0 |
4129 |
SDValue Tmp = DAG.getNode(ISD::OR, dl, NVT, HLAdjusted, ResultHH); |
0 |
| 4130 |
SatMax = DAG.getSetCC(dl, BoolNVT, Tmp, NVTZero, ISD::SETNE); |
0 |
4130 |
SatMax = DAG.getSetCC(dl, BoolNVT, Tmp, NVTZero, ISD::SETNE); |
0 |
| 4131 |
} else if (Scale == NVTSize) { |
0 |
4131 |
} else if (Scale == NVTSize) { |
0 |
| 4132 |
// Overflow happened if (HH != 0). |
--- |
4132 |
// Overflow happened if (HH != 0). |
--- |
| 4133 |
SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETNE); |
0 |
4133 |
SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETNE); |
0 |
| 4134 |
} else if (Scale < VTSize) { |
0 |
4134 |
} else if (Scale < VTSize) { |
0 |
| 4135 |
// Overflow happened if ((HH >> (Scale - NVTSize)) != 0). |
--- |
4135 |
// Overflow happened if ((HH >> (Scale - NVTSize)) != 0). |
--- |
| 4136 |
SDValue HLAdjusted = |
--- |
4136 |
SDValue HLAdjusted = |
--- |
| 4137 |
DAG.getNode(ISD::SRL, dl, NVT, ResultHL, |
0 |
4137 |
DAG.getNode(ISD::SRL, dl, NVT, ResultHL, |
0 |
| 4138 |
DAG.getShiftAmountConstant(Scale - NVTSize, NVT, dl)); |
0 |
4138 |
DAG.getShiftAmountConstant(Scale - NVTSize, NVT, dl)); |
0 |
| 4139 |
SatMax = DAG.getSetCC(dl, BoolNVT, HLAdjusted, NVTZero, ISD::SETNE); |
0 |
4139 |
SatMax = DAG.getSetCC(dl, BoolNVT, HLAdjusted, NVTZero, ISD::SETNE); |
0 |
| 4140 |
} else |
--- |
4140 |
} else |
--- |
| 4141 |
llvm_unreachable("Scale must be less or equal to VTSize for UMULFIXSAT" |
0 |
4141 |
llvm_unreachable("Scale must be less or equal to VTSize for UMULFIXSAT" |
0 |
| 4142 |
"(and saturation can't happen with Scale==VTSize)."); |
--- |
4142 |
"(and saturation can't happen with Scale==VTSize)."); |
--- |
| 4143 |
|
--- |
4143 |
|
--- |
| 4144 |
Hi = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Hi); |
0 |
4144 |
Hi = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Hi); |
0 |
| 4145 |
Lo = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Lo); |
0 |
4145 |
Lo = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Lo); |
0 |
| 4146 |
return; |
0 |
4146 |
return; |
0 |
| 4147 |
} |
--- |
4147 |
} |
--- |
| 4148 |
|
--- |
4148 |
|
--- |
| 4149 |
if (Scale < NVTSize) { |
0 |
4149 |
if (Scale < NVTSize) { |
0 |
| 4150 |
// The number of overflow bits we can check are VTSize - Scale + 1 (we |
--- |
4150 |
// The number of overflow bits we can check are VTSize - Scale + 1 (we |
--- |
| 4151 |
// include the sign bit). If these top bits are > 0, then we overflowed past |
--- |
4151 |
// include the sign bit). If these top bits are > 0, then we overflowed past |
--- |
| 4152 |
// the max value. If these top bits are < -1, then we overflowed past the |
--- |
4152 |
// the max value. If these top bits are < -1, then we overflowed past the |
--- |
| 4153 |
// min value. Otherwise, we did not overflow. |
--- |
4153 |
// min value. Otherwise, we did not overflow. |
--- |
| 4154 |
unsigned OverflowBits = VTSize - Scale + 1; |
0 |
4154 |
unsigned OverflowBits = VTSize - Scale + 1; |
0 |
| 4155 |
assert(OverflowBits <= VTSize && OverflowBits > NVTSize && |
0 |
4155 |
assert(OverflowBits <= VTSize && OverflowBits > NVTSize && |
0 |
| 4156 |
"Extent of overflow bits must start within HL"); |
--- |
4156 |
"Extent of overflow bits must start within HL"); |
--- |
| 4157 |
SDValue HLHiMask = DAG.getConstant( |
0 |
4157 |
SDValue HLHiMask = DAG.getConstant( |
0 |
| 4158 |
APInt::getHighBitsSet(NVTSize, OverflowBits - NVTSize), dl, NVT); |
0 |
4158 |
APInt::getHighBitsSet(NVTSize, OverflowBits - NVTSize), dl, NVT); |
0 |
| 4159 |
SDValue HLLoMask = DAG.getConstant( |
0 |
4159 |
SDValue HLLoMask = DAG.getConstant( |
0 |
| 4160 |
APInt::getLowBitsSet(NVTSize, VTSize - OverflowBits), dl, NVT); |
0 |
4160 |
APInt::getLowBitsSet(NVTSize, VTSize - OverflowBits), dl, NVT); |
0 |
| 4161 |
// We overflow max if HH > 0 or (HH == 0 && HL > HLLoMask). |
--- |
4161 |
// We overflow max if HH > 0 or (HH == 0 && HL > HLLoMask). |
--- |
| 4162 |
SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT); |
0 |
4162 |
SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT); |
0 |
| 4163 |
SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ); |
0 |
4163 |
SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ); |
0 |
| 4164 |
SDValue HLUGT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLLoMask, ISD::SETUGT); |
0 |
4164 |
SDValue HLUGT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLLoMask, ISD::SETUGT); |
0 |
| 4165 |
SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0, |
0 |
4165 |
SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0, |
0 |
| 4166 |
DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLUGT)); |
0 |
4166 |
DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLUGT)); |
0 |
| 4167 |
// We overflow min if HH < -1 or (HH == -1 && HL < HLHiMask). |
--- |
4167 |
// We overflow min if HH < -1 or (HH == -1 && HL < HLHiMask). |
--- |
| 4168 |
SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT); |
0 |
4168 |
SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT); |
0 |
| 4169 |
SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ); |
0 |
4169 |
SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ); |
0 |
| 4170 |
SDValue HLULT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLHiMask, ISD::SETULT); |
0 |
4170 |
SDValue HLULT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLHiMask, ISD::SETULT); |
0 |
| 4171 |
SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT, |
0 |
4171 |
SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT, |
0 |
| 4172 |
DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLULT)); |
0 |
4172 |
DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLULT)); |
0 |
| 4173 |
} else if (Scale == NVTSize) { |
0 |
4173 |
} else if (Scale == NVTSize) { |
0 |
| 4174 |
// We overflow max if HH > 0 or (HH == 0 && HL sign bit is 1). |
--- |
4174 |
// We overflow max if HH > 0 or (HH == 0 && HL sign bit is 1). |
--- |
| 4175 |
SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT); |
0 |
4175 |
SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT); |
0 |
| 4176 |
SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ); |
0 |
4176 |
SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ); |
0 |
| 4177 |
SDValue HLNeg = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETLT); |
0 |
4177 |
SDValue HLNeg = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETLT); |
0 |
| 4178 |
SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0, |
0 |
4178 |
SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0, |
0 |
| 4179 |
DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLNeg)); |
0 |
4179 |
DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLNeg)); |
0 |
| 4180 |
// We overflow min if HH < -1 or (HH == -1 && HL sign bit is 0). |
--- |
4180 |
// We overflow min if HH < -1 or (HH == -1 && HL sign bit is 0). |
--- |
| 4181 |
SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT); |
0 |
4181 |
SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT); |
0 |
| 4182 |
SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ); |
0 |
4182 |
SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ); |
0 |
| 4183 |
SDValue HLPos = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETGE); |
0 |
4183 |
SDValue HLPos = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETGE); |
0 |
| 4184 |
SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT, |
0 |
4184 |
SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT, |
0 |
| 4185 |
DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLPos)); |
0 |
4185 |
DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLPos)); |
0 |
| 4186 |
} else if (Scale < VTSize) { |
0 |
4186 |
} else if (Scale < VTSize) { |
0 |
| 4187 |
// This is similar to the case when we saturate if Scale < NVTSize, but we |
--- |
4187 |
// This is similar to the case when we saturate if Scale < NVTSize, but we |
--- |
| 4188 |
// only need to check HH. |
--- |
4188 |
// only need to check HH. |
--- |
| 4189 |
unsigned OverflowBits = VTSize - Scale + 1; |
0 |
4189 |
unsigned OverflowBits = VTSize - Scale + 1; |
0 |
| 4190 |
SDValue HHHiMask = DAG.getConstant( |
0 |
4190 |
SDValue HHHiMask = DAG.getConstant( |
0 |
| 4191 |
APInt::getHighBitsSet(NVTSize, OverflowBits), dl, NVT); |
0 |
4191 |
APInt::getHighBitsSet(NVTSize, OverflowBits), dl, NVT); |
0 |
| 4192 |
SDValue HHLoMask = DAG.getConstant( |
0 |
4192 |
SDValue HHLoMask = DAG.getConstant( |
0 |
| 4193 |
APInt::getLowBitsSet(NVTSize, NVTSize - OverflowBits), dl, NVT); |
0 |
4193 |
APInt::getLowBitsSet(NVTSize, NVTSize - OverflowBits), dl, NVT); |
0 |
| 4194 |
SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, HHLoMask, ISD::SETGT); |
0 |
4194 |
SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, HHLoMask, ISD::SETGT); |
0 |
| 4195 |
SatMin = DAG.getSetCC(dl, BoolNVT, ResultHH, HHHiMask, ISD::SETLT); |
0 |
4195 |
SatMin = DAG.getSetCC(dl, BoolNVT, ResultHH, HHHiMask, ISD::SETLT); |
0 |
| 4196 |
} else |
--- |
4196 |
} else |
--- |
| 4197 |
llvm_unreachable("Illegal scale for signed fixed point mul."); |
0 |
4197 |
llvm_unreachable("Illegal scale for signed fixed point mul."); |
0 |
| 4198 |
|
--- |
4198 |
|
--- |
| 4199 |
// Saturate to signed maximum. |
--- |
4199 |
// Saturate to signed maximum. |
--- |
| 4200 |
APInt MaxHi = APInt::getSignedMaxValue(NVTSize); |
0 |
4200 |
APInt MaxHi = APInt::getSignedMaxValue(NVTSize); |
0 |
| 4201 |
APInt MaxLo = APInt::getAllOnes(NVTSize); |
0 |
4201 |
APInt MaxLo = APInt::getAllOnes(NVTSize); |
0 |
| 4202 |
Hi = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxHi, dl, NVT), Hi); |
0 |
4202 |
Hi = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxHi, dl, NVT), Hi); |
0 |
| 4203 |
Lo = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxLo, dl, NVT), Lo); |
0 |
4203 |
Lo = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxLo, dl, NVT), Lo); |
0 |
| 4204 |
// Saturate to signed minimum. |
--- |
4204 |
// Saturate to signed minimum. |
--- |
| 4205 |
APInt MinHi = APInt::getSignedMinValue(NVTSize); |
0 |
4205 |
APInt MinHi = APInt::getSignedMinValue(NVTSize); |
0 |
| 4206 |
Hi = DAG.getSelect(dl, NVT, SatMin, DAG.getConstant(MinHi, dl, NVT), Hi); |
0 |
4206 |
Hi = DAG.getSelect(dl, NVT, SatMin, DAG.getConstant(MinHi, dl, NVT), Hi); |
0 |
| 4207 |
Lo = DAG.getSelect(dl, NVT, SatMin, NVTZero, Lo); |
0 |
4207 |
Lo = DAG.getSelect(dl, NVT, SatMin, NVTZero, Lo); |
0 |
| 4208 |
} |
0 |
4208 |
} |
0 |
| 4209 |
|
--- |
4209 |
|
--- |
| 4210 |
void DAGTypeLegalizer::ExpandIntRes_DIVFIX(SDNode *N, SDValue &Lo, |
0 |
4210 |
void DAGTypeLegalizer::ExpandIntRes_DIVFIX(SDNode *N, SDValue &Lo, |
0 |
| 4211 |
SDValue &Hi) { |
--- |
4211 |
SDValue &Hi) { |
--- |
| 4212 |
SDLoc dl(N); |
0 |
4212 |
SDLoc dl(N); |
0 |
| 4213 |
// Try expanding in the existing type first. |
--- |
4213 |
// Try expanding in the existing type first. |
--- |
| 4214 |
SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, N->getOperand(0), |
0 |
4214 |
SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, N->getOperand(0), |
0 |
| 4215 |
N->getOperand(1), |
0 |
4215 |
N->getOperand(1), |
0 |
| 4216 |
N->getConstantOperandVal(2), DAG); |
0 |
4216 |
N->getConstantOperandVal(2), DAG); |
0 |
| 4217 |
|
--- |
4217 |
|
--- |
| 4218 |
if (!Res) |
0 |
4218 |
if (!Res) |
0 |
| 4219 |
Res = earlyExpandDIVFIX(N, N->getOperand(0), N->getOperand(1), |
0 |
4219 |
Res = earlyExpandDIVFIX(N, N->getOperand(0), N->getOperand(1), |
0 |
| 4220 |
N->getConstantOperandVal(2), TLI, DAG); |
0 |
4220 |
N->getConstantOperandVal(2), TLI, DAG); |
0 |
| 4221 |
SplitInteger(Res, Lo, Hi); |
0 |
4221 |
SplitInteger(Res, Lo, Hi); |
0 |
| 4222 |
} |
0 |
4222 |
} |
0 |
| 4223 |
|
--- |
4223 |
|
--- |
| 4224 |
void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node, |
0 |
4224 |
void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node, |
0 |
| 4225 |
SDValue &Lo, SDValue &Hi) { |
--- |
4225 |
SDValue &Lo, SDValue &Hi) { |
--- |
| 4226 |
assert((Node->getOpcode() == ISD::SADDO || Node->getOpcode() == ISD::SSUBO) && |
0 |
4226 |
assert((Node->getOpcode() == ISD::SADDO || Node->getOpcode() == ISD::SSUBO) && |
0 |
| 4227 |
"Node has unexpected Opcode"); |
--- |
4227 |
"Node has unexpected Opcode"); |
--- |
| 4228 |
SDValue LHS = Node->getOperand(0); |
0 |
4228 |
SDValue LHS = Node->getOperand(0); |
0 |
| 4229 |
SDValue RHS = Node->getOperand(1); |
0 |
4229 |
SDValue RHS = Node->getOperand(1); |
0 |
| 4230 |
SDLoc dl(Node); |
0 |
4230 |
SDLoc dl(Node); |
0 |
| 4231 |
|
--- |
4231 |
|
--- |
| 4232 |
SDValue Ovf; |
0 |
4232 |
SDValue Ovf; |
0 |
| 4233 |
|
--- |
4233 |
|
--- |
| 4234 |
bool IsAdd = Node->getOpcode() == ISD::SADDO; |
0 |
4234 |
bool IsAdd = Node->getOpcode() == ISD::SADDO; |
0 |
| 4235 |
unsigned CarryOp = IsAdd ? ISD::SADDO_CARRY : ISD::SSUBO_CARRY; |
0 |
4235 |
unsigned CarryOp = IsAdd ? ISD::SADDO_CARRY : ISD::SSUBO_CARRY; |
0 |
| 4236 |
|
--- |
4236 |
|
--- |
| 4237 |
bool HasCarryOp = TLI.isOperationLegalOrCustom( |
0 |
4237 |
bool HasCarryOp = TLI.isOperationLegalOrCustom( |
0 |
| 4238 |
CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType())); |
0 |
4238 |
CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType())); |
0 |
| 4239 |
|
--- |
4239 |
|
--- |
| 4240 |
if (HasCarryOp) { |
0 |
4240 |
if (HasCarryOp) { |
0 |
| 4241 |
// Expand the subcomponents. |
--- |
4241 |
// Expand the subcomponents. |
--- |
| 4242 |
SDValue LHSL, LHSH, RHSL, RHSH; |
0 |
4242 |
SDValue LHSL, LHSH, RHSL, RHSH; |
0 |
| 4243 |
GetExpandedInteger(LHS, LHSL, LHSH); |
0 |
4243 |
GetExpandedInteger(LHS, LHSL, LHSH); |
0 |
| 4244 |
GetExpandedInteger(RHS, RHSL, RHSH); |
0 |
4244 |
GetExpandedInteger(RHS, RHSL, RHSH); |
0 |
| 4245 |
SDVTList VTList = DAG.getVTList(LHSL.getValueType(), Node->getValueType(1)); |
0 |
4245 |
SDVTList VTList = DAG.getVTList(LHSL.getValueType(), Node->getValueType(1)); |
0 |
| 4246 |
|
--- |
4246 |
|
--- |
| 4247 |
Lo = DAG.getNode(IsAdd ? ISD::UADDO : ISD::USUBO, dl, VTList, {LHSL, RHSL}); |
0 |
4247 |
Lo = DAG.getNode(IsAdd ? ISD::UADDO : ISD::USUBO, dl, VTList, {LHSL, RHSL}); |
0 |
| 4248 |
Hi = DAG.getNode(CarryOp, dl, VTList, { LHSH, RHSH, Lo.getValue(1) }); |
0 |
4248 |
Hi = DAG.getNode(CarryOp, dl, VTList, { LHSH, RHSH, Lo.getValue(1) }); |
0 |
| 4249 |
|
--- |
4249 |
|
--- |
| 4250 |
Ovf = Hi.getValue(1); |
0 |
4250 |
Ovf = Hi.getValue(1); |
0 |
| 4251 |
} else { |
--- |
4251 |
} else { |
--- |
| 4252 |
// Expand the result by simply replacing it with the equivalent |
--- |
4252 |
// Expand the result by simply replacing it with the equivalent |
--- |
| 4253 |
// non-overflow-checking operation. |
--- |
4253 |
// non-overflow-checking operation. |
--- |
| 4254 |
SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ? |
0 |
4254 |
SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ? |
0 |
| 4255 |
ISD::ADD : ISD::SUB, dl, LHS.getValueType(), |
--- |
4255 |
ISD::ADD : ISD::SUB, dl, LHS.getValueType(), |
--- |
| 4256 |
LHS, RHS); |
--- |
4256 |
LHS, RHS); |
--- |
| 4257 |
SplitInteger(Sum, Lo, Hi); |
0 |
4257 |
SplitInteger(Sum, Lo, Hi); |
0 |
| 4258 |
|
--- |
4258 |
|
--- |
| 4259 |
// Compute the overflow. |
--- |
4259 |
// Compute the overflow. |
--- |
| 4260 |
// |
--- |
4260 |
// |
--- |
| 4261 |
// LHSSign -> LHS < 0 |
--- |
4261 |
// LHSSign -> LHS < 0 |
--- |
| 4262 |
// RHSSign -> RHS < 0 |
--- |
4262 |
// RHSSign -> RHS < 0 |
--- |
| 4263 |
// SumSign -> Sum < 0 |
--- |
4263 |
// SumSign -> Sum < 0 |
--- |
| 4264 |
// |
--- |
4264 |
// |
--- |
| 4265 |
// Add: |
--- |
4265 |
// Add: |
--- |
| 4266 |
// Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign) |
--- |
4266 |
// Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign) |
--- |
| 4267 |
// Sub: |
--- |
4267 |
// Sub: |
--- |
| 4268 |
// Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign) |
--- |
4268 |
// Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign) |
--- |
| 4269 |
// |
--- |
4269 |
// |
--- |
| 4270 |
// To get better codegen we can rewrite this by doing bitwise math on |
--- |
4270 |
// To get better codegen we can rewrite this by doing bitwise math on |
--- |
| 4271 |
// the integers and extract the final sign bit at the end. So the |
--- |
4271 |
// the integers and extract the final sign bit at the end. So the |
--- |
| 4272 |
// above becomes: |
--- |
4272 |
// above becomes: |
--- |
| 4273 |
// |
--- |
4273 |
// |
--- |
| 4274 |
// Add: |
--- |
4274 |
// Add: |
--- |
| 4275 |
// Overflow -> (~(LHS ^ RHS) & (LHS ^ Sum)) < 0 |
--- |
4275 |
// Overflow -> (~(LHS ^ RHS) & (LHS ^ Sum)) < 0 |
--- |
| 4276 |
// Sub: |
--- |
4276 |
// Sub: |
--- |
| 4277 |
// Overflow -> ((LHS ^ RHS) & (LHS ^ Sum)) < 0 |
--- |
4277 |
// Overflow -> ((LHS ^ RHS) & (LHS ^ Sum)) < 0 |
--- |
| 4278 |
// |
--- |
4278 |
// |
--- |
| 4279 |
// NOTE: This is different than the expansion we do in expandSADDSUBO |
--- |
4279 |
// NOTE: This is different than the expansion we do in expandSADDSUBO |
--- |
| 4280 |
// because it is more costly to determine the RHS is > 0 for SSUBO with the |
--- |
4280 |
// because it is more costly to determine the RHS is > 0 for SSUBO with the |
--- |
| 4281 |
// integers split. |
--- |
4281 |
// integers split. |
--- |
| 4282 |
EVT VT = LHS.getValueType(); |
0 |
4282 |
EVT VT = LHS.getValueType(); |
0 |
| 4283 |
SDValue SignsMatch = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS); |
0 |
4283 |
SDValue SignsMatch = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS); |
0 |
| 4284 |
if (IsAdd) |
0 |
4284 |
if (IsAdd) |
0 |
| 4285 |
SignsMatch = DAG.getNOT(dl, SignsMatch, VT); |
0 |
4285 |
SignsMatch = DAG.getNOT(dl, SignsMatch, VT); |
0 |
| 4286 |
|
--- |
4286 |
|
--- |
| 4287 |
SDValue SumSignNE = DAG.getNode(ISD::XOR, dl, VT, LHS, Sum); |
0 |
4287 |
SDValue SumSignNE = DAG.getNode(ISD::XOR, dl, VT, LHS, Sum); |
0 |
| 4288 |
Ovf = DAG.getNode(ISD::AND, dl, VT, SignsMatch, SumSignNE); |
0 |
4288 |
Ovf = DAG.getNode(ISD::AND, dl, VT, SignsMatch, SumSignNE); |
0 |
| 4289 |
EVT OType = Node->getValueType(1); |
0 |
4289 |
EVT OType = Node->getValueType(1); |
0 |
| 4290 |
Ovf = DAG.getSetCC(dl, OType, Ovf, DAG.getConstant(0, dl, VT), ISD::SETLT); |
0 |
4290 |
Ovf = DAG.getSetCC(dl, OType, Ovf, DAG.getConstant(0, dl, VT), ISD::SETLT); |
0 |
| 4291 |
} |
--- |
4291 |
} |
--- |
| 4292 |
|
--- |
4292 |
|
--- |
| 4293 |
// Use the calculated overflow everywhere. |
--- |
4293 |
// Use the calculated overflow everywhere. |
--- |
| 4294 |
ReplaceValueWith(SDValue(Node, 1), Ovf); |
0 |
4294 |
ReplaceValueWith(SDValue(Node, 1), Ovf); |
0 |
| 4295 |
} |
0 |
4295 |
} |
0 |
| 4296 |
|
--- |
4296 |
|
--- |
| 4297 |
void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N, |
0 |
4297 |
void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N, |
0 |
| 4298 |
SDValue &Lo, SDValue &Hi) { |
--- |
4298 |
SDValue &Lo, SDValue &Hi) { |
--- |
| 4299 |
EVT VT = N->getValueType(0); |
0 |
4299 |
EVT VT = N->getValueType(0); |
0 |
| 4300 |
SDLoc dl(N); |
0 |
4300 |
SDLoc dl(N); |
0 |
| 4301 |
SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; |
0 |
4301 |
SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; |
0 |
| 4302 |
|
--- |
4302 |
|
--- |
| 4303 |
if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) { |
0 |
4303 |
if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) { |
0 |
| 4304 |
SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops); |
0 |
4304 |
SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops); |
0 |
| 4305 |
SplitInteger(Res.getValue(0), Lo, Hi); |
0 |
4305 |
SplitInteger(Res.getValue(0), Lo, Hi); |
0 |
| 4306 |
return; |
0 |
4306 |
return; |
0 |
| 4307 |
} |
--- |
4307 |
} |
--- |
| 4308 |
|
--- |
4308 |
|
--- |
| 4309 |
RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
0 |
4309 |
RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
0 |
| 4310 |
if (VT == MVT::i16) |
0 |
4310 |
if (VT == MVT::i16) |
0 |
| 4311 |
LC = RTLIB::SDIV_I16; |
0 |
4311 |
LC = RTLIB::SDIV_I16; |
0 |
| 4312 |
else if (VT == MVT::i32) |
0 |
4312 |
else if (VT == MVT::i32) |
0 |
| 4313 |
LC = RTLIB::SDIV_I32; |
0 |
4313 |
LC = RTLIB::SDIV_I32; |
0 |
| 4314 |
else if (VT == MVT::i64) |
0 |
4314 |
else if (VT == MVT::i64) |
0 |
| 4315 |
LC = RTLIB::SDIV_I64; |
0 |
4315 |
LC = RTLIB::SDIV_I64; |
0 |
| 4316 |
else if (VT == MVT::i128) |
0 |
4316 |
else if (VT == MVT::i128) |
0 |
| 4317 |
LC = RTLIB::SDIV_I128; |
0 |
4317 |
LC = RTLIB::SDIV_I128; |
0 |
| 4318 |
assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!"); |
0 |
4318 |
assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!"); |
0 |
| 4319 |
|
--- |
4319 |
|
--- |
| 4320 |
TargetLowering::MakeLibCallOptions CallOptions; |
0 |
4320 |
TargetLowering::MakeLibCallOptions CallOptions; |
0 |
| 4321 |
CallOptions.setSExt(true); |
0 |
4321 |
CallOptions.setSExt(true); |
0 |
| 4322 |
SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); |
0 |
4322 |
SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); |
0 |
| 4323 |
} |
0 |
4323 |
} |
0 |
| 4324 |
|
--- |
4324 |
|
--- |
| 4325 |
void DAGTypeLegalizer::ExpandIntRes_ShiftThroughStack(SDNode *N, SDValue &Lo, |
0 |
4325 |
void DAGTypeLegalizer::ExpandIntRes_ShiftThroughStack(SDNode *N, SDValue &Lo, |
0 |
| 4326 |
SDValue &Hi) { |
--- |
4326 |
SDValue &Hi) { |
--- |
| 4327 |
SDLoc dl(N); |
0 |
4327 |
SDLoc dl(N); |
0 |
| 4328 |
SDValue Shiftee = N->getOperand(0); |
0 |
4328 |
SDValue Shiftee = N->getOperand(0); |
0 |
| 4329 |
EVT VT = Shiftee.getValueType(); |
0 |
4329 |
EVT VT = Shiftee.getValueType(); |
0 |
| 4330 |
SDValue ShAmt = N->getOperand(1); |
0 |
4330 |
SDValue ShAmt = N->getOperand(1); |
0 |
| 4331 |
EVT ShAmtVT = ShAmt.getValueType(); |
0 |
4331 |
EVT ShAmtVT = ShAmt.getValueType(); |
0 |
| 4332 |
|
--- |
4332 |
|
--- |
| 4333 |
// This legalization is optimal when the shift is by a multiple of byte width, |
--- |
4333 |
// This legalization is optimal when the shift is by a multiple of byte width, |
--- |
| 4334 |
// %x * 8 <-> %x << 3 so 3 low bits should be be known zero. |
--- |
4334 |
// %x * 8 <-> %x << 3 so 3 low bits should be be known zero. |
--- |
| 4335 |
bool ShiftByByteMultiple = |
--- |
4335 |
bool ShiftByByteMultiple = |
--- |
| 4336 |
DAG.computeKnownBits(ShAmt).countMinTrailingZeros() >= 3; |
0 |
4336 |
DAG.computeKnownBits(ShAmt).countMinTrailingZeros() >= 3; |
0 |
| 4337 |
|
--- |
4337 |
|
--- |
| 4338 |
// If we can't do it as one step, we'll have two uses of shift amount, |
--- |
4338 |
// If we can't do it as one step, we'll have two uses of shift amount, |
--- |
| 4339 |
// and thus must freeze it. |
--- |
4339 |
// and thus must freeze it. |
--- |
| 4340 |
if (!ShiftByByteMultiple) |
0 |
4340 |
if (!ShiftByByteMultiple) |
0 |
| 4341 |
ShAmt = DAG.getFreeze(ShAmt); |
0 |
4341 |
ShAmt = DAG.getFreeze(ShAmt); |
0 |
| 4342 |
|
--- |
4342 |
|
--- |
| 4343 |
unsigned VTBitWidth = VT.getScalarSizeInBits(); |
0 |
4343 |
unsigned VTBitWidth = VT.getScalarSizeInBits(); |
0 |
| 4344 |
assert(VTBitWidth % 8 == 0 && "Shifting a not byte multiple value?"); |
0 |
4344 |
assert(VTBitWidth % 8 == 0 && "Shifting a not byte multiple value?"); |
0 |
| 4345 |
unsigned VTByteWidth = VTBitWidth / 8; |
0 |
4345 |
unsigned VTByteWidth = VTBitWidth / 8; |
0 |
| 4346 |
assert(isPowerOf2_32(VTByteWidth) && |
0 |
4346 |
assert(isPowerOf2_32(VTByteWidth) && |
0 |
| 4347 |
"Shiftee type size is not a power of two!"); |
--- |
4347 |
"Shiftee type size is not a power of two!"); |
--- |
| 4348 |
unsigned StackSlotByteWidth = 2 * VTByteWidth; |
0 |
4348 |
unsigned StackSlotByteWidth = 2 * VTByteWidth; |
0 |
| 4349 |
unsigned StackSlotBitWidth = 8 * StackSlotByteWidth; |
0 |
4349 |
unsigned StackSlotBitWidth = 8 * StackSlotByteWidth; |
0 |
| 4350 |
EVT StackSlotVT = EVT::getIntegerVT(*DAG.getContext(), StackSlotBitWidth); |
0 |
4350 |
EVT StackSlotVT = EVT::getIntegerVT(*DAG.getContext(), StackSlotBitWidth); |
0 |
| 4351 |
|
--- |
4351 |
|
--- |
| 4352 |
// Get a temporary stack slot 2x the width of our VT. |
--- |
4352 |
// Get a temporary stack slot 2x the width of our VT. |
--- |
| 4353 |
// FIXME: reuse stack slots? |
--- |
4353 |
// FIXME: reuse stack slots? |
--- |
| 4354 |
// FIXME: should we be more picky about alignment? |
--- |
4354 |
// FIXME: should we be more picky about alignment? |
--- |
| 4355 |
Align StackSlotAlignment(1); |
0 |
4355 |
Align StackSlotAlignment(1); |
0 |
| 4356 |
SDValue StackPtr = DAG.CreateStackTemporary( |
0 |
4356 |
SDValue StackPtr = DAG.CreateStackTemporary( |
0 |
| 4357 |
TypeSize::getFixed(StackSlotByteWidth), StackSlotAlignment); |
--- |
4357 |
TypeSize::getFixed(StackSlotByteWidth), StackSlotAlignment); |
--- |
| 4358 |
EVT PtrTy = StackPtr.getValueType(); |
0 |
4358 |
EVT PtrTy = StackPtr.getValueType(); |
0 |
| 4359 |
SDValue Ch = DAG.getEntryNode(); |
0 |
4359 |
SDValue Ch = DAG.getEntryNode(); |
0 |
| 4360 |
|
--- |
4360 |
|
--- |
| 4361 |
MachinePointerInfo StackPtrInfo = MachinePointerInfo::getFixedStack( |
0 |
4361 |
MachinePointerInfo StackPtrInfo = MachinePointerInfo::getFixedStack( |
0 |
| 4362 |
DAG.getMachineFunction(), |
0 |
4362 |
DAG.getMachineFunction(), |
0 |
| 4363 |
cast(StackPtr.getNode())->getIndex()); |
--- |
4363 |
cast(StackPtr.getNode())->getIndex()); |
--- |
| 4364 |
|
--- |
4364 |
|
--- |
| 4365 |
// Extend the value, that is being shifted, to the entire stack slot's width. |
--- |
4365 |
// Extend the value, that is being shifted, to the entire stack slot's width. |
--- |
| 4366 |
SDValue Init; |
0 |
4366 |
SDValue Init; |
0 |
| 4367 |
if (N->getOpcode() != ISD::SHL) { |
0 |
4367 |
if (N->getOpcode() != ISD::SHL) { |
0 |
| 4368 |
unsigned WideningOpc = |
--- |
4368 |
unsigned WideningOpc = |
--- |
| 4369 |
N->getOpcode() == ISD::SRA ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; |
0 |
4369 |
N->getOpcode() == ISD::SRA ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; |
0 |
| 4370 |
Init = DAG.getNode(WideningOpc, dl, StackSlotVT, Shiftee); |
0 |
4370 |
Init = DAG.getNode(WideningOpc, dl, StackSlotVT, Shiftee); |
0 |
| 4371 |
} else { |
--- |
4371 |
} else { |
--- |
| 4372 |
// For left-shifts, pad the Shiftee's LSB with zeros to twice it's width. |
--- |
4372 |
// For left-shifts, pad the Shiftee's LSB with zeros to twice it's width. |
--- |
| 4373 |
SDValue AllZeros = DAG.getConstant(0, dl, VT); |
0 |
4373 |
SDValue AllZeros = DAG.getConstant(0, dl, VT); |
0 |
| 4374 |
Init = DAG.getNode(ISD::BUILD_PAIR, dl, StackSlotVT, AllZeros, Shiftee); |
0 |
4374 |
Init = DAG.getNode(ISD::BUILD_PAIR, dl, StackSlotVT, AllZeros, Shiftee); |
0 |
| 4375 |
} |
--- |
4375 |
} |
--- |
| 4376 |
// And spill it into the stack slot. |
--- |
4376 |
// And spill it into the stack slot. |
--- |
| 4377 |
Ch = DAG.getStore(Ch, dl, Init, StackPtr, StackPtrInfo, StackSlotAlignment); |
0 |
4377 |
Ch = DAG.getStore(Ch, dl, Init, StackPtr, StackPtrInfo, StackSlotAlignment); |
0 |
| 4378 |
|
--- |
4378 |
|
--- |
| 4379 |
// Now, compute the full-byte offset into stack slot from where we can load. |
--- |
4379 |
// Now, compute the full-byte offset into stack slot from where we can load. |
--- |
| 4380 |
// We have shift amount, which is in bits, but in multiples of byte. |
--- |
4380 |
// We have shift amount, which is in bits, but in multiples of byte. |
--- |
| 4381 |
// So just divide by CHAR_BIT. |
--- |
4381 |
// So just divide by CHAR_BIT. |
--- |
| 4382 |
SDNodeFlags Flags; |
0 |
4382 |
SDNodeFlags Flags; |
0 |
| 4383 |
if (ShiftByByteMultiple) |
0 |
4383 |
if (ShiftByByteMultiple) |
0 |
| 4384 |
Flags.setExact(true); |
0 |
4384 |
Flags.setExact(true); |
0 |
| 4385 |
SDValue ByteOffset = DAG.getNode(ISD::SRL, dl, ShAmtVT, ShAmt, |
0 |
4385 |
SDValue ByteOffset = DAG.getNode(ISD::SRL, dl, ShAmtVT, ShAmt, |
0 |
| 4386 |
DAG.getConstant(3, dl, ShAmtVT), Flags); |
0 |
4386 |
DAG.getConstant(3, dl, ShAmtVT), Flags); |
0 |
| 4387 |
// And clamp it, because OOB load is an immediate UB, |
--- |
4387 |
// And clamp it, because OOB load is an immediate UB, |
--- |
| 4388 |
// while shift overflow would have *just* been poison. |
--- |
4388 |
// while shift overflow would have *just* been poison. |
--- |
| 4389 |
ByteOffset = DAG.getNode(ISD::AND, dl, ShAmtVT, ByteOffset, |
0 |
4389 |
ByteOffset = DAG.getNode(ISD::AND, dl, ShAmtVT, ByteOffset, |
0 |
| 4390 |
DAG.getConstant(VTByteWidth - 1, dl, ShAmtVT)); |
0 |
4390 |
DAG.getConstant(VTByteWidth - 1, dl, ShAmtVT)); |
0 |
| 4391 |
// We have exactly two strategies on indexing into stack slot here: |
--- |
4391 |
// We have exactly two strategies on indexing into stack slot here: |
--- |
| 4392 |
// 1. upwards starting from the beginning of the slot |
--- |
4392 |
// 1. upwards starting from the beginning of the slot |
--- |
| 4393 |
// 2. downwards starting from the middle of the slot |
--- |
4393 |
// 2. downwards starting from the middle of the slot |
--- |
| 4394 |
// On little-endian machine, we pick 1. for right shifts and 2. for left-shift |
--- |
4394 |
// On little-endian machine, we pick 1. for right shifts and 2. for left-shift |
--- |
| 4395 |
// and vice versa on big-endian machine. |
--- |
4395 |
// and vice versa on big-endian machine. |
--- |
| 4396 |
bool WillIndexUpwards = N->getOpcode() != ISD::SHL; |
0 |
4396 |
bool WillIndexUpwards = N->getOpcode() != ISD::SHL; |
0 |
| 4397 |
if (DAG.getDataLayout().isBigEndian()) |
0 |
4397 |
if (DAG.getDataLayout().isBigEndian()) |
0 |
| 4398 |
WillIndexUpwards = !WillIndexUpwards; |
0 |
4398 |
WillIndexUpwards = !WillIndexUpwards; |
0 |
| 4399 |
|
--- |
4399 |
|
--- |
| 4400 |
SDValue AdjStackPtr; |
0 |
4400 |
SDValue AdjStackPtr; |
0 |
| 4401 |
if (WillIndexUpwards) { |
0 |
4401 |
if (WillIndexUpwards) { |
0 |
| 4402 |
AdjStackPtr = StackPtr; |
0 |
4402 |
AdjStackPtr = StackPtr; |
0 |
| 4403 |
} else { |
--- |
4403 |
} else { |
--- |
| 4404 |
AdjStackPtr = DAG.getMemBasePlusOffset( |
0 |
4404 |
AdjStackPtr = DAG.getMemBasePlusOffset( |
0 |
| 4405 |
StackPtr, DAG.getConstant(VTByteWidth, dl, PtrTy), dl); |
0 |
4405 |
StackPtr, DAG.getConstant(VTByteWidth, dl, PtrTy), dl); |
0 |
| 4406 |
ByteOffset = DAG.getNegative(ByteOffset, dl, ShAmtVT); |
0 |
4406 |
ByteOffset = DAG.getNegative(ByteOffset, dl, ShAmtVT); |
0 |
| 4407 |
} |
--- |
4407 |
} |
--- |
| 4408 |
|
--- |
4408 |
|
--- |
| 4409 |
// Get the pointer somewhere into the stack slot from which we need to load. |
--- |
4409 |
// Get the pointer somewhere into the stack slot from which we need to load. |
--- |
| 4410 |
ByteOffset = DAG.getSExtOrTrunc(ByteOffset, dl, PtrTy); |
0 |
4410 |
ByteOffset = DAG.getSExtOrTrunc(ByteOffset, dl, PtrTy); |
0 |
| 4411 |
AdjStackPtr = DAG.getMemBasePlusOffset(AdjStackPtr, ByteOffset, dl); |
0 |
4411 |
AdjStackPtr = DAG.getMemBasePlusOffset(AdjStackPtr, ByteOffset, dl); |
0 |
| 4412 |
|
--- |
4412 |
|
--- |
| 4413 |
// And load it! While the load is not legal, legalizing it is obvious. |
--- |
4413 |
// And load it! While the load is not legal, legalizing it is obvious. |
--- |
| 4414 |
SDValue Res = DAG.getLoad( |
0 |
4414 |
SDValue Res = DAG.getLoad( |
0 |
| 4415 |
VT, dl, Ch, AdjStackPtr, |
--- |
4415 |
VT, dl, Ch, AdjStackPtr, |
--- |
| 4416 |
MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), Align(1)); |
0 |
4416 |
MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), Align(1)); |
0 |
| 4417 |
// We've performed the shift by a CHAR_BIT * [_ShAmt / CHAR_BIT_] |
--- |
4417 |
// We've performed the shift by a CHAR_BIT * [_ShAmt / CHAR_BIT_] |
--- |
| 4418 |
|
--- |
4418 |
|
--- |
| 4419 |
// If we may still have a less-than-CHAR_BIT to shift by, do so now. |
--- |
4419 |
// If we may still have a less-than-CHAR_BIT to shift by, do so now. |
--- |
| 4420 |
if (!ShiftByByteMultiple) { |
0 |
4420 |
if (!ShiftByByteMultiple) { |
0 |
| 4421 |
SDValue ShAmtRem = DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt, |
0 |
4421 |
SDValue ShAmtRem = DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt, |
0 |
| 4422 |
DAG.getConstant(7, dl, ShAmtVT)); |
0 |
4422 |
DAG.getConstant(7, dl, ShAmtVT)); |
0 |
| 4423 |
Res = DAG.getNode(N->getOpcode(), dl, VT, Res, ShAmtRem); |
0 |
4423 |
Res = DAG.getNode(N->getOpcode(), dl, VT, Res, ShAmtRem); |
0 |
| 4424 |
} |
--- |
4424 |
} |
--- |
| 4425 |
|
--- |
4425 |
|
--- |
| 4426 |
// Finally, split the computed value. |
--- |
4426 |
// Finally, split the computed value. |
--- |
| 4427 |
SplitInteger(Res, Lo, Hi); |
0 |
4427 |
SplitInteger(Res, Lo, Hi); |
0 |
| 4428 |
} |
0 |
4428 |
} |
0 |
| 4429 |
|
--- |
4429 |
|
--- |
| 4430 |
void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N, |
0 |
4430 |
void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N, |
0 |
| 4431 |
SDValue &Lo, SDValue &Hi) { |
--- |
4431 |
SDValue &Lo, SDValue &Hi) { |
--- |
| 4432 |
EVT VT = N->getValueType(0); |
0 |
4432 |
EVT VT = N->getValueType(0); |
0 |
| 4433 |
SDLoc dl(N); |
0 |
4433 |
SDLoc dl(N); |
0 |
| 4434 |
|
--- |
4434 |
|
--- |
| 4435 |
// If we can emit an efficient shift operation, do so now. Check to see if |
--- |
4435 |
// If we can emit an efficient shift operation, do so now. Check to see if |
--- |
| 4436 |
// the RHS is a constant. |
--- |
4436 |
// the RHS is a constant. |
--- |
| 4437 |
if (ConstantSDNode *CN = dyn_cast(N->getOperand(1))) |
0 |
4437 |
if (ConstantSDNode *CN = dyn_cast(N->getOperand(1))) |
0 |
| 4438 |
return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi); |
0 |
4438 |
return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi); |
0 |
| 4439 |
|
--- |
4439 |
|
--- |
| 4440 |
// If we can determine that the high bit of the shift is zero or one, even if |
--- |
4440 |
// If we can determine that the high bit of the shift is zero or one, even if |
--- |
| 4441 |
// the low bits are variable, emit this shift in an optimized form. |
--- |
4441 |
// the low bits are variable, emit this shift in an optimized form. |
--- |
| 4442 |
if (ExpandShiftWithKnownAmountBit(N, Lo, Hi)) |
0 |
4442 |
if (ExpandShiftWithKnownAmountBit(N, Lo, Hi)) |
0 |
| 4443 |
return; |
0 |
4443 |
return; |
0 |
| 4444 |
|
--- |
4444 |
|
--- |
| 4445 |
// If this target supports shift_PARTS, use it. First, map to the _PARTS opc. |
--- |
4445 |
// If this target supports shift_PARTS, use it. First, map to the _PARTS opc. |
--- |
| 4446 |
unsigned PartsOpc; |
--- |
4446 |
unsigned PartsOpc; |
--- |
| 4447 |
if (N->getOpcode() == ISD::SHL) { |
0 |
4447 |
if (N->getOpcode() == ISD::SHL) { |
0 |
| 4448 |
PartsOpc = ISD::SHL_PARTS; |
0 |
4448 |
PartsOpc = ISD::SHL_PARTS; |
0 |
| 4449 |
} else if (N->getOpcode() == ISD::SRL) { |
0 |
4449 |
} else if (N->getOpcode() == ISD::SRL) { |
0 |
| 4450 |
PartsOpc = ISD::SRL_PARTS; |
0 |
4450 |
PartsOpc = ISD::SRL_PARTS; |
0 |
| 4451 |
} else { |
--- |
4451 |
} else { |
--- |
| 4452 |
assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); |
0 |
4452 |
assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); |
0 |
| 4453 |
PartsOpc = ISD::SRA_PARTS; |
0 |
4453 |
PartsOpc = ISD::SRA_PARTS; |
0 |
| 4454 |
} |
--- |
4454 |
} |
--- |
| 4455 |
|
--- |
4455 |
|
--- |
| 4456 |
// Next check to see if the target supports this SHL_PARTS operation or if it |
--- |
4456 |
// Next check to see if the target supports this SHL_PARTS operation or if it |
--- |
| 4457 |
// will custom expand it. Don't lower this to SHL_PARTS when we optimise for |
--- |
4457 |
// will custom expand it. Don't lower this to SHL_PARTS when we optimise for |
--- |
| 4458 |
// size, but create a libcall instead. |
--- |
4458 |
// size, but create a libcall instead. |
--- |
| 4459 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); |
0 |
4459 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); |
0 |
| 4460 |
TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT); |
0 |
4460 |
TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT); |
0 |
| 4461 |
const bool LegalOrCustom = |
--- |
4461 |
const bool LegalOrCustom = |
--- |
| 4462 |
(Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) || |
0 |
4462 |
(Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) || |
0 |
| 4463 |
Action == TargetLowering::Custom; |
0 |
4463 |
Action == TargetLowering::Custom; |
0 |
| 4464 |
|
--- |
4464 |
|
--- |
| 4465 |
unsigned ExpansionFactor = 1; |
0 |
4465 |
unsigned ExpansionFactor = 1; |
0 |
| 4466 |
// That VT->NVT expansion is one step. But will we re-expand NVT? |
--- |
4466 |
// That VT->NVT expansion is one step. But will we re-expand NVT? |
--- |
| 4467 |
for (EVT TmpVT = NVT;;) { |
0 |
4467 |
for (EVT TmpVT = NVT;;) { |
0 |
| 4468 |
EVT NewTMPVT = TLI.getTypeToTransformTo(*DAG.getContext(), TmpVT); |
0 |
4468 |
EVT NewTMPVT = TLI.getTypeToTransformTo(*DAG.getContext(), TmpVT); |
0 |
| 4469 |
if (NewTMPVT == TmpVT) |
0 |
4469 |
if (NewTMPVT == TmpVT) |
0 |
| 4470 |
break; |
0 |
4470 |
break; |
0 |
| 4471 |
TmpVT = NewTMPVT; |
0 |
4471 |
TmpVT = NewTMPVT; |
0 |
| 4472 |
++ExpansionFactor; |
0 |
4472 |
++ExpansionFactor; |
0 |
| 4473 |
} |
0 |
4473 |
} |
0 |
| 4474 |
|
--- |
4474 |
|
--- |
| 4475 |
TargetLowering::ShiftLegalizationStrategy S = |
--- |
4475 |
TargetLowering::ShiftLegalizationStrategy S = |
--- |
| 4476 |
TLI.preferredShiftLegalizationStrategy(DAG, N, ExpansionFactor); |
0 |
4476 |
TLI.preferredShiftLegalizationStrategy(DAG, N, ExpansionFactor); |
0 |
| 4477 |
|
--- |
4477 |
|
--- |
| 4478 |
if (S == TargetLowering::ShiftLegalizationStrategy::ExpandThroughStack) |
0 |
4478 |
if (S == TargetLowering::ShiftLegalizationStrategy::ExpandThroughStack) |
0 |
| 4479 |
return ExpandIntRes_ShiftThroughStack(N, Lo, Hi); |
0 |
4479 |
return ExpandIntRes_ShiftThroughStack(N, Lo, Hi); |
0 |
| 4480 |
|
--- |
4480 |
|
--- |
| 4481 |
if (LegalOrCustom && |
0 |
4481 |
if (LegalOrCustom && |
0 |
| 4482 |
S != TargetLowering::ShiftLegalizationStrategy::LowerToLibcall) { |
--- |
4482 |
S != TargetLowering::ShiftLegalizationStrategy::LowerToLibcall) { |
--- |
| 4483 |
// Expand the subcomponents. |
--- |
4483 |
// Expand the subcomponents. |
--- |
| 4484 |
SDValue LHSL, LHSH; |
0 |
4484 |
SDValue LHSL, LHSH; |
0 |
| 4485 |
GetExpandedInteger(N->getOperand(0), LHSL, LHSH); |
0 |
4485 |
GetExpandedInteger(N->getOperand(0), LHSL, LHSH); |
0 |
| 4486 |
EVT VT = LHSL.getValueType(); |
0 |
4486 |
EVT VT = LHSL.getValueType(); |
0 |
| 4487 |
|
--- |
4487 |
|
--- |
| 4488 |
// If the shift amount operand is coming from a vector legalization it may |
--- |
4488 |
// If the shift amount operand is coming from a vector legalization it may |
--- |
| 4489 |
// have an illegal type. Fix that first by casting the operand, otherwise |
--- |
4489 |
// have an illegal type. Fix that first by casting the operand, otherwise |
--- |
| 4490 |
// the new SHL_PARTS operation would need further legalization. |
--- |
4490 |
// the new SHL_PARTS operation would need further legalization. |
--- |
| 4491 |
SDValue ShiftOp = N->getOperand(1); |
0 |
4491 |
SDValue ShiftOp = N->getOperand(1); |
0 |
| 4492 |
EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); |
0 |
4492 |
EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); |
0 |
| 4493 |
if (ShiftOp.getValueType() != ShiftTy) |
0 |
4493 |
if (ShiftOp.getValueType() != ShiftTy) |
0 |
| 4494 |
ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy); |
0 |
4494 |
ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy); |
0 |
| 4495 |
|
--- |
4495 |
|
--- |
| 4496 |
SDValue Ops[] = { LHSL, LHSH, ShiftOp }; |
0 |
4496 |
SDValue Ops[] = { LHSL, LHSH, ShiftOp }; |
0 |
| 4497 |
Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops); |
0 |
4497 |
Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops); |
0 |
| 4498 |
Hi = Lo.getValue(1); |
0 |
4498 |
Hi = Lo.getValue(1); |
0 |
| 4499 |
return; |
0 |
4499 |
return; |
0 |
| 4500 |
} |
--- |
4500 |
} |
--- |
| 4501 |
|
--- |
4501 |
|
--- |
| 4502 |
// Otherwise, emit a libcall. |
--- |
4502 |
// Otherwise, emit a libcall. |
--- |
| 4503 |
RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
0 |
4503 |
RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
0 |
| 4504 |
bool isSigned; |
--- |
4504 |
bool isSigned; |
--- |
| 4505 |
if (N->getOpcode() == ISD::SHL) { |
0 |
4505 |
if (N->getOpcode() == ISD::SHL) { |
0 |
| 4506 |
isSigned = false; /*sign irrelevant*/ |
0 |
4506 |
isSigned = false; /*sign irrelevant*/ |
0 |
| 4507 |
if (VT == MVT::i16) |
0 |
4507 |
if (VT == MVT::i16) |
0 |
| 4508 |
LC = RTLIB::SHL_I16; |
0 |
4508 |
LC = RTLIB::SHL_I16; |
0 |
| 4509 |
else if (VT == MVT::i32) |
0 |
4509 |
else if (VT == MVT::i32) |
0 |
| 4510 |
LC = RTLIB::SHL_I32; |
0 |
4510 |
LC = RTLIB::SHL_I32; |
0 |
| 4511 |
else if (VT == MVT::i64) |
0 |
4511 |
else if (VT == MVT::i64) |
0 |
| 4512 |
LC = RTLIB::SHL_I64; |
0 |
4512 |
LC = RTLIB::SHL_I64; |
0 |
| 4513 |
else if (VT == MVT::i128) |
0 |
4513 |
else if (VT == MVT::i128) |
0 |
| 4514 |
LC = RTLIB::SHL_I128; |
0 |
4514 |
LC = RTLIB::SHL_I128; |
0 |
| 4515 |
} else if (N->getOpcode() == ISD::SRL) { |
0 |
4515 |
} else if (N->getOpcode() == ISD::SRL) { |
0 |
| 4516 |
isSigned = false; |
0 |
4516 |
isSigned = false; |
0 |
| 4517 |
if (VT == MVT::i16) |
0 |
4517 |
if (VT == MVT::i16) |
0 |
| 4518 |
LC = RTLIB::SRL_I16; |
0 |
4518 |
LC = RTLIB::SRL_I16; |
0 |
| 4519 |
else if (VT == MVT::i32) |
0 |
4519 |
else if (VT == MVT::i32) |
0 |
| 4520 |
LC = RTLIB::SRL_I32; |
0 |
4520 |
LC = RTLIB::SRL_I32; |
0 |
| 4521 |
else if (VT == MVT::i64) |
0 |
4521 |
else if (VT == MVT::i64) |
0 |
| 4522 |
LC = RTLIB::SRL_I64; |
0 |
4522 |
LC = RTLIB::SRL_I64; |
0 |
| 4523 |
else if (VT == MVT::i128) |
0 |
4523 |
else if (VT == MVT::i128) |
0 |
| 4524 |
LC = RTLIB::SRL_I128; |
0 |
4524 |
LC = RTLIB::SRL_I128; |
0 |
| 4525 |
} else { |
--- |
4525 |
} else { |
--- |
| 4526 |
assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); |
0 |
4526 |
assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); |
0 |
| 4527 |
isSigned = true; |
0 |
4527 |
isSigned = true; |
0 |
| 4528 |
if (VT == MVT::i16) |
0 |
4528 |
if (VT == MVT::i16) |
0 |
| 4529 |
LC = RTLIB::SRA_I16; |
0 |
4529 |
LC = RTLIB::SRA_I16; |
0 |
| 4530 |
else if (VT == MVT::i32) |
0 |
4530 |
else if (VT == MVT::i32) |
0 |
| 4531 |
LC = RTLIB::SRA_I32; |
0 |
4531 |
LC = RTLIB::SRA_I32; |
0 |
| 4532 |
else if (VT == MVT::i64) |
0 |
4532 |
else if (VT == MVT::i64) |
0 |
| 4533 |
LC = RTLIB::SRA_I64; |
0 |
4533 |
LC = RTLIB::SRA_I64; |
0 |
| 4534 |
else if (VT == MVT::i128) |
0 |
4534 |
else if (VT == MVT::i128) |
0 |
| 4535 |
LC = RTLIB::SRA_I128; |
0 |
4535 |
LC = RTLIB::SRA_I128; |
0 |
| 4536 |
} |
--- |
4536 |
} |
--- |
| 4537 |
|
--- |
4537 |
|
--- |
| 4538 |
if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) { |
0 |
4538 |
if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) { |
0 |
| 4539 |
EVT ShAmtTy = |
--- |
4539 |
EVT ShAmtTy = |
--- |
| 4540 |
EVT::getIntegerVT(*DAG.getContext(), DAG.getLibInfo().getIntSize()); |
0 |
4540 |
EVT::getIntegerVT(*DAG.getContext(), DAG.getLibInfo().getIntSize()); |
0 |
| 4541 |
SDValue ShAmt = DAG.getZExtOrTrunc(N->getOperand(1), dl, ShAmtTy); |
0 |
4541 |
SDValue ShAmt = DAG.getZExtOrTrunc(N->getOperand(1), dl, ShAmtTy); |
0 |
| 4542 |
SDValue Ops[2] = {N->getOperand(0), ShAmt}; |
0 |
4542 |
SDValue Ops[2] = {N->getOperand(0), ShAmt}; |
0 |
| 4543 |
TargetLowering::MakeLibCallOptions CallOptions; |
0 |
4543 |
TargetLowering::MakeLibCallOptions CallOptions; |
0 |
| 4544 |
CallOptions.setSExt(isSigned); |
0 |
4544 |
CallOptions.setSExt(isSigned); |
0 |
| 4545 |
SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); |
0 |
4545 |
SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); |
0 |
| 4546 |
return; |
0 |
4546 |
return; |
0 |
| 4547 |
} |
--- |
4547 |
} |
--- |
| 4548 |
|
--- |
4548 |
|
--- |
| 4549 |
if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi)) |
0 |
4549 |
if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi)) |
0 |
| 4550 |
llvm_unreachable("Unsupported shift!"); |
0 |
4550 |
llvm_unreachable("Unsupported shift!"); |
0 |
| 4551 |
} |
0 |
4551 |
} |
0 |
| 4552 |
|
--- |
4552 |
|
--- |
| 4553 |
void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N, |
0 |
4553 |
void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N, |
0 |
| 4554 |
SDValue &Lo, SDValue &Hi) { |
--- |
4554 |
SDValue &Lo, SDValue &Hi) { |
--- |
| 4555 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
4555 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
| 4556 |
SDLoc dl(N); |
0 |
4556 |
SDLoc dl(N); |
0 |
| 4557 |
SDValue Op = N->getOperand(0); |
0 |
4557 |
SDValue Op = N->getOperand(0); |
0 |
| 4558 |
if (Op.getValueType().bitsLE(NVT)) { |
0 |
4558 |
if (Op.getValueType().bitsLE(NVT)) { |
0 |
| 4559 |
// The low part is sign extension of the input (degenerates to a copy). |
--- |
4559 |
// The low part is sign extension of the input (degenerates to a copy). |
--- |
| 4560 |
Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0)); |
0 |
4560 |
Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0)); |
0 |
| 4561 |
// The high part is obtained by SRA'ing all but one of the bits of low part. |
--- |
4561 |
// The high part is obtained by SRA'ing all but one of the bits of low part. |
--- |
| 4562 |
unsigned LoSize = NVT.getSizeInBits(); |
0 |
4562 |
unsigned LoSize = NVT.getSizeInBits(); |
0 |
| 4563 |
Hi = DAG.getNode( |
0 |
4563 |
Hi = DAG.getNode( |
0 |
| 4564 |
ISD::SRA, dl, NVT, Lo, |
--- |
4564 |
ISD::SRA, dl, NVT, Lo, |
--- |
| 4565 |
DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout()))); |
0 |
4565 |
DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout()))); |
0 |
| 4566 |
} else { |
--- |
4566 |
} else { |
--- |
| 4567 |
// For example, extension of an i48 to an i64. The operand type necessarily |
--- |
4567 |
// For example, extension of an i48 to an i64. The operand type necessarily |
--- |
| 4568 |
// promotes to the result type, so will end up being expanded too. |
--- |
4568 |
// promotes to the result type, so will end up being expanded too. |
--- |
| 4569 |
assert(getTypeAction(Op.getValueType()) == |
0 |
4569 |
assert(getTypeAction(Op.getValueType()) == |
0 |
| 4570 |
TargetLowering::TypePromoteInteger && |
--- |
4570 |
TargetLowering::TypePromoteInteger && |
--- |
| 4571 |
"Only know how to promote this result!"); |
--- |
4571 |
"Only know how to promote this result!"); |
--- |
| 4572 |
SDValue Res = GetPromotedInteger(Op); |
0 |
4572 |
SDValue Res = GetPromotedInteger(Op); |
0 |
| 4573 |
assert(Res.getValueType() == N->getValueType(0) && |
0 |
4573 |
assert(Res.getValueType() == N->getValueType(0) && |
0 |
| 4574 |
"Operand over promoted?"); |
--- |
4574 |
"Operand over promoted?"); |
--- |
| 4575 |
// Split the promoted operand. This will simplify when it is expanded. |
--- |
4575 |
// Split the promoted operand. This will simplify when it is expanded. |
--- |
| 4576 |
SplitInteger(Res, Lo, Hi); |
0 |
4576 |
SplitInteger(Res, Lo, Hi); |
0 |
| 4577 |
unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits(); |
0 |
4577 |
unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits(); |
0 |
| 4578 |
Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, |
0 |
4578 |
Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, |
0 |
| 4579 |
DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), |
0 |
4579 |
DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), |
0 |
| 4580 |
ExcessBits))); |
--- |
4580 |
ExcessBits))); |
--- |
| 4581 |
} |
--- |
4581 |
} |
--- |
| 4582 |
} |
0 |
4582 |
} |
0 |
| 4583 |
|
--- |
4583 |
|
--- |
| 4584 |
void DAGTypeLegalizer:: |
0 |
4584 |
void DAGTypeLegalizer:: |
0 |
| 4585 |
ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) { |
--- |
4585 |
ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) { |
--- |
| 4586 |
SDLoc dl(N); |
0 |
4586 |
SDLoc dl(N); |
0 |
| 4587 |
GetExpandedInteger(N->getOperand(0), Lo, Hi); |
0 |
4587 |
GetExpandedInteger(N->getOperand(0), Lo, Hi); |
0 |
| 4588 |
EVT EVT = cast(N->getOperand(1))->getVT(); |
0 |
4588 |
EVT EVT = cast(N->getOperand(1))->getVT(); |
0 |
| 4589 |
|
--- |
4589 |
|
--- |
| 4590 |
if (EVT.bitsLE(Lo.getValueType())) { |
0 |
4590 |
if (EVT.bitsLE(Lo.getValueType())) { |
0 |
| 4591 |
// sext_inreg the low part if needed. |
--- |
4591 |
// sext_inreg the low part if needed. |
--- |
| 4592 |
Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo, |
0 |
4592 |
Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo, |
0 |
| 4593 |
N->getOperand(1)); |
0 |
4593 |
N->getOperand(1)); |
0 |
| 4594 |
|
--- |
4594 |
|
--- |
| 4595 |
// The high part gets the sign extension from the lo-part. This handles |
--- |
4595 |
// The high part gets the sign extension from the lo-part. This handles |
--- |
| 4596 |
// things like sextinreg V:i64 from i8. |
--- |
4596 |
// things like sextinreg V:i64 from i8. |
--- |
| 4597 |
Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo, |
0 |
4597 |
Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo, |
0 |
| 4598 |
DAG.getConstant(Hi.getValueSizeInBits() - 1, dl, |
0 |
4598 |
DAG.getConstant(Hi.getValueSizeInBits() - 1, dl, |
0 |
| 4599 |
TLI.getPointerTy(DAG.getDataLayout()))); |
0 |
4599 |
TLI.getPointerTy(DAG.getDataLayout()))); |
0 |
| 4600 |
} else { |
--- |
4600 |
} else { |
--- |
| 4601 |
// For example, extension of an i48 to an i64. Leave the low part alone, |
--- |
4601 |
// For example, extension of an i48 to an i64. Leave the low part alone, |
--- |
| 4602 |
// sext_inreg the high part. |
--- |
4602 |
// sext_inreg the high part. |
--- |
| 4603 |
unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits(); |
0 |
4603 |
unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits(); |
0 |
| 4604 |
Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, |
0 |
4604 |
Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, |
0 |
| 4605 |
DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), |
0 |
4605 |
DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), |
0 |
| 4606 |
ExcessBits))); |
--- |
4606 |
ExcessBits))); |
--- |
| 4607 |
} |
--- |
4607 |
} |
--- |
| 4608 |
} |
0 |
4608 |
} |
0 |
| 4609 |
|
--- |
4609 |
|
--- |
| 4610 |
void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N, |
0 |
4610 |
void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N, |
0 |
| 4611 |
SDValue &Lo, SDValue &Hi) { |
--- |
4611 |
SDValue &Lo, SDValue &Hi) { |
--- |
| 4612 |
EVT VT = N->getValueType(0); |
0 |
4612 |
EVT VT = N->getValueType(0); |
0 |
| 4613 |
SDLoc dl(N); |
0 |
4613 |
SDLoc dl(N); |
0 |
| 4614 |
SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; |
0 |
4614 |
SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; |
0 |
| 4615 |
|
--- |
4615 |
|
--- |
| 4616 |
if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) { |
0 |
4616 |
if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) { |
0 |
| 4617 |
SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops); |
0 |
4617 |
SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops); |
0 |
| 4618 |
SplitInteger(Res.getValue(1), Lo, Hi); |
0 |
4618 |
SplitInteger(Res.getValue(1), Lo, Hi); |
0 |
| 4619 |
return; |
0 |
4619 |
return; |
0 |
| 4620 |
} |
--- |
4620 |
} |
--- |
| 4621 |
|
--- |
4621 |
|
--- |
| 4622 |
RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
0 |
4622 |
RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
0 |
| 4623 |
if (VT == MVT::i16) |
0 |
4623 |
if (VT == MVT::i16) |
0 |
| 4624 |
LC = RTLIB::SREM_I16; |
0 |
4624 |
LC = RTLIB::SREM_I16; |
0 |
| 4625 |
else if (VT == MVT::i32) |
0 |
4625 |
else if (VT == MVT::i32) |
0 |
| 4626 |
LC = RTLIB::SREM_I32; |
0 |
4626 |
LC = RTLIB::SREM_I32; |
0 |
| 4627 |
else if (VT == MVT::i64) |
0 |
4627 |
else if (VT == MVT::i64) |
0 |
| 4628 |
LC = RTLIB::SREM_I64; |
0 |
4628 |
LC = RTLIB::SREM_I64; |
0 |
| 4629 |
else if (VT == MVT::i128) |
0 |
4629 |
else if (VT == MVT::i128) |
0 |
| 4630 |
LC = RTLIB::SREM_I128; |
0 |
4630 |
LC = RTLIB::SREM_I128; |
0 |
| 4631 |
assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!"); |
0 |
4631 |
assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!"); |
0 |
| 4632 |
|
--- |
4632 |
|
--- |
| 4633 |
TargetLowering::MakeLibCallOptions CallOptions; |
0 |
4633 |
TargetLowering::MakeLibCallOptions CallOptions; |
0 |
| 4634 |
CallOptions.setSExt(true); |
0 |
4634 |
CallOptions.setSExt(true); |
0 |
| 4635 |
SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); |
0 |
4635 |
SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); |
0 |
| 4636 |
} |
0 |
4636 |
} |
0 |
| 4637 |
|
--- |
4637 |
|
--- |
| 4638 |
void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N, |
0 |
4638 |
void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N, |
0 |
| 4639 |
SDValue &Lo, SDValue &Hi) { |
--- |
4639 |
SDValue &Lo, SDValue &Hi) { |
--- |
| 4640 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
4640 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
| 4641 |
SDLoc dl(N); |
0 |
4641 |
SDLoc dl(N); |
0 |
| 4642 |
Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0)); |
0 |
4642 |
Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0)); |
0 |
| 4643 |
Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(), |
0 |
4643 |
Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(), |
0 |
| 4644 |
N->getOperand(0), |
0 |
4644 |
N->getOperand(0), |
0 |
| 4645 |
DAG.getConstant(NVT.getSizeInBits(), dl, |
0 |
4645 |
DAG.getConstant(NVT.getSizeInBits(), dl, |
0 |
| 4646 |
TLI.getPointerTy(DAG.getDataLayout()))); |
0 |
4646 |
TLI.getPointerTy(DAG.getDataLayout()))); |
0 |
| 4647 |
Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi); |
0 |
4647 |
Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi); |
0 |
| 4648 |
} |
0 |
4648 |
} |
0 |
| 4649 |
|
--- |
4649 |
|
--- |
| 4650 |
void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N, |
0 |
4650 |
void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N, |
0 |
| 4651 |
SDValue &Lo, SDValue &Hi) { |
--- |
4651 |
SDValue &Lo, SDValue &Hi) { |
--- |
| 4652 |
EVT VT = N->getValueType(0); |
0 |
4652 |
EVT VT = N->getValueType(0); |
0 |
| 4653 |
SDLoc dl(N); |
0 |
4653 |
SDLoc dl(N); |
0 |
| 4654 |
|
--- |
4654 |
|
--- |
| 4655 |
if (N->getOpcode() == ISD::UMULO) { |
0 |
4655 |
if (N->getOpcode() == ISD::UMULO) { |
0 |
| 4656 |
// This section expands the operation into the following sequence of |
--- |
4656 |
// This section expands the operation into the following sequence of |
--- |
| 4657 |
// instructions. `iNh` here refers to a type which has half the bit width of |
--- |
4657 |
// instructions. `iNh` here refers to a type which has half the bit width of |
--- |
| 4658 |
// the type the original operation operated on. |
--- |
4658 |
// the type the original operation operated on. |
--- |
| 4659 |
// |
--- |
4659 |
// |
--- |
| 4660 |
// %0 = %LHS.HI != 0 && %RHS.HI != 0 |
--- |
4660 |
// %0 = %LHS.HI != 0 && %RHS.HI != 0 |
--- |
| 4661 |
// %1 = { iNh, i1 } @umul.with.overflow.iNh(iNh %LHS.HI, iNh %RHS.LO) |
--- |
4661 |
// %1 = { iNh, i1 } @umul.with.overflow.iNh(iNh %LHS.HI, iNh %RHS.LO) |
--- |
| 4662 |
// %2 = { iNh, i1 } @umul.with.overflow.iNh(iNh %RHS.HI, iNh %LHS.LO) |
--- |
4662 |
// %2 = { iNh, i1 } @umul.with.overflow.iNh(iNh %RHS.HI, iNh %LHS.LO) |
--- |
| 4663 |
// %3 = mul nuw iN (%LHS.LOW as iN), (%RHS.LOW as iN) |
--- |
4663 |
// %3 = mul nuw iN (%LHS.LOW as iN), (%RHS.LOW as iN) |
--- |
| 4664 |
// %4 = add iNh %1.0, %2.0 as iN |
--- |
4664 |
// %4 = add iNh %1.0, %2.0 as iN |
--- |
| 4665 |
// %5 = { iNh, i1 } @uadd.with.overflow.iNh(iNh %4, iNh %3.HIGH) |
--- |
4665 |
// %5 = { iNh, i1 } @uadd.with.overflow.iNh(iNh %4, iNh %3.HIGH) |
--- |
| 4666 |
// |
--- |
4666 |
// |
--- |
| 4667 |
// %lo = %3.LO |
--- |
4667 |
// %lo = %3.LO |
--- |
| 4668 |
// %hi = %5.0 |
--- |
4668 |
// %hi = %5.0 |
--- |
| 4669 |
// %ovf = %0 || %1.1 || %2.1 || %5.1 |
--- |
4669 |
// %ovf = %0 || %1.1 || %2.1 || %5.1 |
--- |
| 4670 |
SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); |
0 |
4670 |
SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); |
0 |
| 4671 |
SDValue LHSHigh, LHSLow, RHSHigh, RHSLow; |
0 |
4671 |
SDValue LHSHigh, LHSLow, RHSHigh, RHSLow; |
0 |
| 4672 |
GetExpandedInteger(LHS, LHSLow, LHSHigh); |
0 |
4672 |
GetExpandedInteger(LHS, LHSLow, LHSHigh); |
0 |
| 4673 |
GetExpandedInteger(RHS, RHSLow, RHSHigh); |
0 |
4673 |
GetExpandedInteger(RHS, RHSLow, RHSHigh); |
0 |
| 4674 |
EVT HalfVT = LHSLow.getValueType(); |
0 |
4674 |
EVT HalfVT = LHSLow.getValueType(); |
0 |
| 4675 |
EVT BitVT = N->getValueType(1); |
0 |
4675 |
EVT BitVT = N->getValueType(1); |
0 |
| 4676 |
SDVTList VTHalfWithO = DAG.getVTList(HalfVT, BitVT); |
0 |
4676 |
SDVTList VTHalfWithO = DAG.getVTList(HalfVT, BitVT); |
0 |
| 4677 |
|
--- |
4677 |
|
--- |
| 4678 |
SDValue HalfZero = DAG.getConstant(0, dl, HalfVT); |
0 |
4678 |
SDValue HalfZero = DAG.getConstant(0, dl, HalfVT); |
0 |
| 4679 |
SDValue Overflow = DAG.getNode(ISD::AND, dl, BitVT, |
0 |
4679 |
SDValue Overflow = DAG.getNode(ISD::AND, dl, BitVT, |
0 |
| 4680 |
DAG.getSetCC(dl, BitVT, LHSHigh, HalfZero, ISD::SETNE), |
0 |
4680 |
DAG.getSetCC(dl, BitVT, LHSHigh, HalfZero, ISD::SETNE), |
0 |
| 4681 |
DAG.getSetCC(dl, BitVT, RHSHigh, HalfZero, ISD::SETNE)); |
0 |
4681 |
DAG.getSetCC(dl, BitVT, RHSHigh, HalfZero, ISD::SETNE)); |
0 |
| 4682 |
|
--- |
4682 |
|
--- |
| 4683 |
SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, LHSHigh, RHSLow); |
0 |
4683 |
SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, LHSHigh, RHSLow); |
0 |
| 4684 |
Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, One.getValue(1)); |
0 |
4684 |
Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, One.getValue(1)); |
0 |
| 4685 |
|
--- |
4685 |
|
--- |
| 4686 |
SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, RHSHigh, LHSLow); |
0 |
4686 |
SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, RHSHigh, LHSLow); |
0 |
| 4687 |
Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Two.getValue(1)); |
0 |
4687 |
Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Two.getValue(1)); |
0 |
| 4688 |
|
--- |
4688 |
|
--- |
| 4689 |
SDValue HighSum = DAG.getNode(ISD::ADD, dl, HalfVT, One, Two); |
0 |
4689 |
SDValue HighSum = DAG.getNode(ISD::ADD, dl, HalfVT, One, Two); |
0 |
| 4690 |
|
--- |
4690 |
|
--- |
| 4691 |
// Cannot use `UMUL_LOHI` directly, because some 32-bit targets (ARM) do not |
--- |
4691 |
// Cannot use `UMUL_LOHI` directly, because some 32-bit targets (ARM) do not |
--- |
| 4692 |
// know how to expand `i64,i64 = umul_lohi a, b` and abort (why isn’t this |
--- |
4692 |
// know how to expand `i64,i64 = umul_lohi a, b` and abort (why isn’t this |
--- |
| 4693 |
// operation recursively legalized?). |
--- |
4693 |
// operation recursively legalized?). |
--- |
| 4694 |
// |
--- |
4694 |
// |
--- |
| 4695 |
// Many backends understand this pattern and will convert into LOHI |
--- |
4695 |
// Many backends understand this pattern and will convert into LOHI |
--- |
| 4696 |
// themselves, if applicable. |
--- |
4696 |
// themselves, if applicable. |
--- |
| 4697 |
SDValue Three = DAG.getNode(ISD::MUL, dl, VT, |
0 |
4697 |
SDValue Three = DAG.getNode(ISD::MUL, dl, VT, |
0 |
| 4698 |
DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LHSLow), |
0 |
4698 |
DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LHSLow), |
0 |
| 4699 |
DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RHSLow)); |
0 |
4699 |
DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RHSLow)); |
0 |
| 4700 |
SplitInteger(Three, Lo, Hi); |
0 |
4700 |
SplitInteger(Three, Lo, Hi); |
0 |
| 4701 |
|
--- |
4701 |
|
--- |
| 4702 |
Hi = DAG.getNode(ISD::UADDO, dl, VTHalfWithO, Hi, HighSum); |
0 |
4702 |
Hi = DAG.getNode(ISD::UADDO, dl, VTHalfWithO, Hi, HighSum); |
0 |
| 4703 |
Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Hi.getValue(1)); |
0 |
4703 |
Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Hi.getValue(1)); |
0 |
| 4704 |
ReplaceValueWith(SDValue(N, 1), Overflow); |
0 |
4704 |
ReplaceValueWith(SDValue(N, 1), Overflow); |
0 |
| 4705 |
return; |
0 |
4705 |
return; |
0 |
| 4706 |
} |
--- |
4706 |
} |
--- |
| 4707 |
|
--- |
4707 |
|
--- |
| 4708 |
Type *RetTy = VT.getTypeForEVT(*DAG.getContext()); |
0 |
4708 |
Type *RetTy = VT.getTypeForEVT(*DAG.getContext()); |
0 |
| 4709 |
EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); |
0 |
4709 |
EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); |
0 |
| 4710 |
Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext()); |
0 |
4710 |
Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext()); |
0 |
| 4711 |
|
--- |
4711 |
|
--- |
| 4712 |
// Replace this with a libcall that will check overflow. |
--- |
4712 |
// Replace this with a libcall that will check overflow. |
--- |
| 4713 |
RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
0 |
4713 |
RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
0 |
| 4714 |
if (VT == MVT::i32) |
0 |
4714 |
if (VT == MVT::i32) |
0 |
| 4715 |
LC = RTLIB::MULO_I32; |
0 |
4715 |
LC = RTLIB::MULO_I32; |
0 |
| 4716 |
else if (VT == MVT::i64) |
0 |
4716 |
else if (VT == MVT::i64) |
0 |
| 4717 |
LC = RTLIB::MULO_I64; |
0 |
4717 |
LC = RTLIB::MULO_I64; |
0 |
| 4718 |
else if (VT == MVT::i128) |
0 |
4718 |
else if (VT == MVT::i128) |
0 |
| 4719 |
LC = RTLIB::MULO_I128; |
0 |
4719 |
LC = RTLIB::MULO_I128; |
0 |
| 4720 |
|
--- |
4720 |
|
--- |
| 4721 |
// If we don't have the libcall or if the function we are compiling is the |
--- |
4721 |
// If we don't have the libcall or if the function we are compiling is the |
--- |
| 4722 |
// implementation of the expected libcall (avoid inf-loop), expand inline. |
--- |
4722 |
// implementation of the expected libcall (avoid inf-loop), expand inline. |
--- |
| 4723 |
if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC) || |
0 |
4723 |
if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC) || |
0 |
| 4724 |
TLI.getLibcallName(LC) == DAG.getMachineFunction().getName()) { |
0 |
4724 |
TLI.getLibcallName(LC) == DAG.getMachineFunction().getName()) { |
0 |
| 4725 |
// FIXME: This is not an optimal expansion, but better than crashing. |
--- |
4725 |
// FIXME: This is not an optimal expansion, but better than crashing. |
--- |
| 4726 |
EVT WideVT = |
--- |
4726 |
EVT WideVT = |
--- |
| 4727 |
EVT::getIntegerVT(*DAG.getContext(), VT.getScalarSizeInBits() * 2); |
0 |
4727 |
EVT::getIntegerVT(*DAG.getContext(), VT.getScalarSizeInBits() * 2); |
0 |
| 4728 |
SDValue LHS = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, N->getOperand(0)); |
0 |
4728 |
SDValue LHS = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, N->getOperand(0)); |
0 |
| 4729 |
SDValue RHS = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, N->getOperand(1)); |
0 |
4729 |
SDValue RHS = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, N->getOperand(1)); |
0 |
| 4730 |
SDValue Mul = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS); |
0 |
4730 |
SDValue Mul = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS); |
0 |
| 4731 |
SDValue MulLo, MulHi; |
0 |
4731 |
SDValue MulLo, MulHi; |
0 |
| 4732 |
SplitInteger(Mul, MulLo, MulHi); |
0 |
4732 |
SplitInteger(Mul, MulLo, MulHi); |
0 |
| 4733 |
SDValue SRA = |
--- |
4733 |
SDValue SRA = |
--- |
| 4734 |
DAG.getNode(ISD::SRA, dl, VT, MulLo, |
0 |
4734 |
DAG.getNode(ISD::SRA, dl, VT, MulLo, |
0 |
| 4735 |
DAG.getConstant(VT.getScalarSizeInBits() - 1, dl, VT)); |
0 |
4735 |
DAG.getConstant(VT.getScalarSizeInBits() - 1, dl, VT)); |
0 |
| 4736 |
SDValue Overflow = |
--- |
4736 |
SDValue Overflow = |
--- |
| 4737 |
DAG.getSetCC(dl, N->getValueType(1), MulHi, SRA, ISD::SETNE); |
0 |
4737 |
DAG.getSetCC(dl, N->getValueType(1), MulHi, SRA, ISD::SETNE); |
0 |
| 4738 |
SplitInteger(MulLo, Lo, Hi); |
0 |
4738 |
SplitInteger(MulLo, Lo, Hi); |
0 |
| 4739 |
ReplaceValueWith(SDValue(N, 1), Overflow); |
0 |
4739 |
ReplaceValueWith(SDValue(N, 1), Overflow); |
0 |
| 4740 |
return; |
0 |
4740 |
return; |
0 |
| 4741 |
} |
--- |
4741 |
} |
--- |
| 4742 |
|
--- |
4742 |
|
--- |
| 4743 |
SDValue Temp = DAG.CreateStackTemporary(PtrVT); |
0 |
4743 |
SDValue Temp = DAG.CreateStackTemporary(PtrVT); |
0 |
| 4744 |
// Temporary for the overflow value, default it to zero. |
--- |
4744 |
// Temporary for the overflow value, default it to zero. |
--- |
| 4745 |
SDValue Chain = |
--- |
4745 |
SDValue Chain = |
--- |
| 4746 |
DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp, |
0 |
4746 |
DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp, |
0 |
| 4747 |
MachinePointerInfo()); |
--- |
4747 |
MachinePointerInfo()); |
--- |
| 4748 |
|
--- |
4748 |
|
--- |
| 4749 |
TargetLowering::ArgListTy Args; |
0 |
4749 |
TargetLowering::ArgListTy Args; |
0 |
| 4750 |
TargetLowering::ArgListEntry Entry; |
0 |
4750 |
TargetLowering::ArgListEntry Entry; |
0 |
| 4751 |
for (const SDValue &Op : N->op_values()) { |
0 |
4751 |
for (const SDValue &Op : N->op_values()) { |
0 |
| 4752 |
EVT ArgVT = Op.getValueType(); |
0 |
4752 |
EVT ArgVT = Op.getValueType(); |
0 |
| 4753 |
Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); |
0 |
4753 |
Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); |
0 |
| 4754 |
Entry.Node = Op; |
0 |
4754 |
Entry.Node = Op; |
0 |
| 4755 |
Entry.Ty = ArgTy; |
0 |
4755 |
Entry.Ty = ArgTy; |
0 |
| 4756 |
Entry.IsSExt = true; |
0 |
4756 |
Entry.IsSExt = true; |
0 |
| 4757 |
Entry.IsZExt = false; |
0 |
4757 |
Entry.IsZExt = false; |
0 |
| 4758 |
Args.push_back(Entry); |
0 |
4758 |
Args.push_back(Entry); |
0 |
| 4759 |
} |
--- |
4759 |
} |
--- |
| 4760 |
|
--- |
4760 |
|
--- |
| 4761 |
// Also pass the address of the overflow check. |
--- |
4761 |
// Also pass the address of the overflow check. |
--- |
| 4762 |
Entry.Node = Temp; |
0 |
4762 |
Entry.Node = Temp; |
0 |
| 4763 |
Entry.Ty = PtrTy->getPointerTo(); |
0 |
4763 |
Entry.Ty = PtrTy->getPointerTo(); |
0 |
| 4764 |
Entry.IsSExt = true; |
0 |
4764 |
Entry.IsSExt = true; |
0 |
| 4765 |
Entry.IsZExt = false; |
0 |
4765 |
Entry.IsZExt = false; |
0 |
| 4766 |
Args.push_back(Entry); |
0 |
4766 |
Args.push_back(Entry); |
0 |
| 4767 |
|
--- |
4767 |
|
--- |
| 4768 |
SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT); |
0 |
4768 |
SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT); |
0 |
| 4769 |
|
--- |
4769 |
|
--- |
| 4770 |
TargetLowering::CallLoweringInfo CLI(DAG); |
0 |
4770 |
TargetLowering::CallLoweringInfo CLI(DAG); |
0 |
| 4771 |
CLI.setDebugLoc(dl) |
0 |
4771 |
CLI.setDebugLoc(dl) |
0 |
| 4772 |
.setChain(Chain) |
0 |
4772 |
.setChain(Chain) |
0 |
| 4773 |
.setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args)) |
0 |
4773 |
.setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args)) |
0 |
| 4774 |
.setSExtResult(); |
0 |
4774 |
.setSExtResult(); |
0 |
| 4775 |
|
--- |
4775 |
|
--- |
| 4776 |
std::pair CallInfo = TLI.LowerCallTo(CLI); |
0 |
4776 |
std::pair CallInfo = TLI.LowerCallTo(CLI); |
0 |
| 4777 |
|
--- |
4777 |
|
--- |
| 4778 |
SplitInteger(CallInfo.first, Lo, Hi); |
0 |
4778 |
SplitInteger(CallInfo.first, Lo, Hi); |
0 |
| 4779 |
SDValue Temp2 = |
--- |
4779 |
SDValue Temp2 = |
--- |
| 4780 |
DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo()); |
0 |
4780 |
DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo()); |
0 |
| 4781 |
SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2, |
0 |
4781 |
SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2, |
0 |
| 4782 |
DAG.getConstant(0, dl, PtrVT), |
0 |
4782 |
DAG.getConstant(0, dl, PtrVT), |
0 |
| 4783 |
ISD::SETNE); |
--- |
4783 |
ISD::SETNE); |
--- |
| 4784 |
// Use the overflow from the libcall everywhere. |
--- |
4784 |
// Use the overflow from the libcall everywhere. |
--- |
| 4785 |
ReplaceValueWith(SDValue(N, 1), Ofl); |
0 |
4785 |
ReplaceValueWith(SDValue(N, 1), Ofl); |
0 |
| 4786 |
} |
0 |
4786 |
} |
0 |
| 4787 |
|
--- |
4787 |
|
--- |
| 4788 |
void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N, |
0 |
4788 |
void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N, |
0 |
| 4789 |
SDValue &Lo, SDValue &Hi) { |
--- |
4789 |
SDValue &Lo, SDValue &Hi) { |
--- |
| 4790 |
EVT VT = N->getValueType(0); |
0 |
4790 |
EVT VT = N->getValueType(0); |
0 |
| 4791 |
SDLoc dl(N); |
0 |
4791 |
SDLoc dl(N); |
0 |
| 4792 |
SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; |
0 |
4792 |
SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; |
0 |
| 4793 |
|
--- |
4793 |
|
--- |
| 4794 |
if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) { |
0 |
4794 |
if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) { |
0 |
| 4795 |
SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops); |
0 |
4795 |
SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops); |
0 |
| 4796 |
SplitInteger(Res.getValue(0), Lo, Hi); |
0 |
4796 |
SplitInteger(Res.getValue(0), Lo, Hi); |
0 |
| 4797 |
return; |
0 |
4797 |
return; |
0 |
| 4798 |
} |
--- |
4798 |
} |
--- |
| 4799 |
|
--- |
4799 |
|
--- |
| 4800 |
// Try to expand UDIV by constant. |
--- |
4800 |
// Try to expand UDIV by constant. |
--- |
| 4801 |
if (isa(N->getOperand(1))) { |
0 |
4801 |
if (isa(N->getOperand(1))) { |
0 |
| 4802 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
4802 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
| 4803 |
// Only if the new type is legal. |
--- |
4803 |
// Only if the new type is legal. |
--- |
| 4804 |
if (isTypeLegal(NVT)) { |
0 |
4804 |
if (isTypeLegal(NVT)) { |
0 |
| 4805 |
SDValue InL, InH; |
0 |
4805 |
SDValue InL, InH; |
0 |
| 4806 |
GetExpandedInteger(N->getOperand(0), InL, InH); |
0 |
4806 |
GetExpandedInteger(N->getOperand(0), InL, InH); |
0 |
| 4807 |
SmallVector Result; |
0 |
4807 |
SmallVector Result; |
0 |
| 4808 |
if (TLI.expandDIVREMByConstant(N, Result, NVT, DAG, InL, InH)) { |
0 |
4808 |
if (TLI.expandDIVREMByConstant(N, Result, NVT, DAG, InL, InH)) { |
0 |
| 4809 |
Lo = Result[0]; |
0 |
4809 |
Lo = Result[0]; |
0 |
| 4810 |
Hi = Result[1]; |
0 |
4810 |
Hi = Result[1]; |
0 |
| 4811 |
return; |
0 |
4811 |
return; |
0 |
| 4812 |
} |
--- |
4812 |
} |
--- |
| 4813 |
} |
0 |
4813 |
} |
0 |
| 4814 |
} |
--- |
4814 |
} |
--- |
| 4815 |
|
--- |
4815 |
|
--- |
| 4816 |
RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
0 |
4816 |
RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
0 |
| 4817 |
if (VT == MVT::i16) |
0 |
4817 |
if (VT == MVT::i16) |
0 |
| 4818 |
LC = RTLIB::UDIV_I16; |
0 |
4818 |
LC = RTLIB::UDIV_I16; |
0 |
| 4819 |
else if (VT == MVT::i32) |
0 |
4819 |
else if (VT == MVT::i32) |
0 |
| 4820 |
LC = RTLIB::UDIV_I32; |
0 |
4820 |
LC = RTLIB::UDIV_I32; |
0 |
| 4821 |
else if (VT == MVT::i64) |
0 |
4821 |
else if (VT == MVT::i64) |
0 |
| 4822 |
LC = RTLIB::UDIV_I64; |
0 |
4822 |
LC = RTLIB::UDIV_I64; |
0 |
| 4823 |
else if (VT == MVT::i128) |
0 |
4823 |
else if (VT == MVT::i128) |
0 |
| 4824 |
LC = RTLIB::UDIV_I128; |
0 |
4824 |
LC = RTLIB::UDIV_I128; |
0 |
| 4825 |
assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!"); |
0 |
4825 |
assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!"); |
0 |
| 4826 |
|
--- |
4826 |
|
--- |
| 4827 |
TargetLowering::MakeLibCallOptions CallOptions; |
0 |
4827 |
TargetLowering::MakeLibCallOptions CallOptions; |
0 |
| 4828 |
SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); |
0 |
4828 |
SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); |
0 |
| 4829 |
} |
0 |
4829 |
} |
0 |
| 4830 |
|
--- |
4830 |
|
--- |
| 4831 |
void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N, |
0 |
4831 |
void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N, |
0 |
| 4832 |
SDValue &Lo, SDValue &Hi) { |
--- |
4832 |
SDValue &Lo, SDValue &Hi) { |
--- |
| 4833 |
EVT VT = N->getValueType(0); |
0 |
4833 |
EVT VT = N->getValueType(0); |
0 |
| 4834 |
SDLoc dl(N); |
0 |
4834 |
SDLoc dl(N); |
0 |
| 4835 |
SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; |
0 |
4835 |
SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; |
0 |
| 4836 |
|
--- |
4836 |
|
--- |
| 4837 |
if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) { |
0 |
4837 |
if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) { |
0 |
| 4838 |
SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops); |
0 |
4838 |
SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops); |
0 |
| 4839 |
SplitInteger(Res.getValue(1), Lo, Hi); |
0 |
4839 |
SplitInteger(Res.getValue(1), Lo, Hi); |
0 |
| 4840 |
return; |
0 |
4840 |
return; |
0 |
| 4841 |
} |
--- |
4841 |
} |
--- |
| 4842 |
|
--- |
4842 |
|
--- |
| 4843 |
// Try to expand UREM by constant. |
--- |
4843 |
// Try to expand UREM by constant. |
--- |
| 4844 |
if (isa(N->getOperand(1))) { |
0 |
4844 |
if (isa(N->getOperand(1))) { |
0 |
| 4845 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
4845 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
| 4846 |
// Only if the new type is legal. |
--- |
4846 |
// Only if the new type is legal. |
--- |
| 4847 |
if (isTypeLegal(NVT)) { |
0 |
4847 |
if (isTypeLegal(NVT)) { |
0 |
| 4848 |
SDValue InL, InH; |
0 |
4848 |
SDValue InL, InH; |
0 |
| 4849 |
GetExpandedInteger(N->getOperand(0), InL, InH); |
0 |
4849 |
GetExpandedInteger(N->getOperand(0), InL, InH); |
0 |
| 4850 |
SmallVector Result; |
0 |
4850 |
SmallVector Result; |
0 |
| 4851 |
if (TLI.expandDIVREMByConstant(N, Result, NVT, DAG, InL, InH)) { |
0 |
4851 |
if (TLI.expandDIVREMByConstant(N, Result, NVT, DAG, InL, InH)) { |
0 |
| 4852 |
Lo = Result[0]; |
0 |
4852 |
Lo = Result[0]; |
0 |
| 4853 |
Hi = Result[1]; |
0 |
4853 |
Hi = Result[1]; |
0 |
| 4854 |
return; |
0 |
4854 |
return; |
0 |
| 4855 |
} |
--- |
4855 |
} |
--- |
| 4856 |
} |
0 |
4856 |
} |
0 |
| 4857 |
} |
--- |
4857 |
} |
--- |
| 4858 |
|
--- |
4858 |
|
--- |
| 4859 |
RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
0 |
4859 |
RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
0 |
| 4860 |
if (VT == MVT::i16) |
0 |
4860 |
if (VT == MVT::i16) |
0 |
| 4861 |
LC = RTLIB::UREM_I16; |
0 |
4861 |
LC = RTLIB::UREM_I16; |
0 |
| 4862 |
else if (VT == MVT::i32) |
0 |
4862 |
else if (VT == MVT::i32) |
0 |
| 4863 |
LC = RTLIB::UREM_I32; |
0 |
4863 |
LC = RTLIB::UREM_I32; |
0 |
| 4864 |
else if (VT == MVT::i64) |
0 |
4864 |
else if (VT == MVT::i64) |
0 |
| 4865 |
LC = RTLIB::UREM_I64; |
0 |
4865 |
LC = RTLIB::UREM_I64; |
0 |
| 4866 |
else if (VT == MVT::i128) |
0 |
4866 |
else if (VT == MVT::i128) |
0 |
| 4867 |
LC = RTLIB::UREM_I128; |
0 |
4867 |
LC = RTLIB::UREM_I128; |
0 |
| 4868 |
assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!"); |
0 |
4868 |
assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!"); |
0 |
| 4869 |
|
--- |
4869 |
|
--- |
| 4870 |
TargetLowering::MakeLibCallOptions CallOptions; |
0 |
4870 |
TargetLowering::MakeLibCallOptions CallOptions; |
0 |
| 4871 |
SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); |
0 |
4871 |
SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); |
0 |
| 4872 |
} |
0 |
4872 |
} |
0 |
| 4873 |
|
--- |
4873 |
|
--- |
| 4874 |
void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N, |
0 |
4874 |
void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N, |
0 |
| 4875 |
SDValue &Lo, SDValue &Hi) { |
--- |
4875 |
SDValue &Lo, SDValue &Hi) { |
--- |
| 4876 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
4876 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
| 4877 |
SDLoc dl(N); |
0 |
4877 |
SDLoc dl(N); |
0 |
| 4878 |
SDValue Op = N->getOperand(0); |
0 |
4878 |
SDValue Op = N->getOperand(0); |
0 |
| 4879 |
if (Op.getValueType().bitsLE(NVT)) { |
0 |
4879 |
if (Op.getValueType().bitsLE(NVT)) { |
0 |
| 4880 |
// The low part is zero extension of the input (degenerates to a copy). |
--- |
4880 |
// The low part is zero extension of the input (degenerates to a copy). |
--- |
| 4881 |
Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0)); |
0 |
4881 |
Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0)); |
0 |
| 4882 |
Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero. |
0 |
4882 |
Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero. |
0 |
| 4883 |
} else { |
--- |
4883 |
} else { |
--- |
| 4884 |
// For example, extension of an i48 to an i64. The operand type necessarily |
--- |
4884 |
// For example, extension of an i48 to an i64. The operand type necessarily |
--- |
| 4885 |
// promotes to the result type, so will end up being expanded too. |
--- |
4885 |
// promotes to the result type, so will end up being expanded too. |
--- |
| 4886 |
assert(getTypeAction(Op.getValueType()) == |
0 |
4886 |
assert(getTypeAction(Op.getValueType()) == |
0 |
| 4887 |
TargetLowering::TypePromoteInteger && |
--- |
4887 |
TargetLowering::TypePromoteInteger && |
--- |
| 4888 |
"Only know how to promote this result!"); |
--- |
4888 |
"Only know how to promote this result!"); |
--- |
| 4889 |
SDValue Res = GetPromotedInteger(Op); |
0 |
4889 |
SDValue Res = GetPromotedInteger(Op); |
0 |
| 4890 |
assert(Res.getValueType() == N->getValueType(0) && |
0 |
4890 |
assert(Res.getValueType() == N->getValueType(0) && |
0 |
| 4891 |
"Operand over promoted?"); |
--- |
4891 |
"Operand over promoted?"); |
--- |
| 4892 |
// Split the promoted operand. This will simplify when it is expanded. |
--- |
4892 |
// Split the promoted operand. This will simplify when it is expanded. |
--- |
| 4893 |
SplitInteger(Res, Lo, Hi); |
0 |
4893 |
SplitInteger(Res, Lo, Hi); |
0 |
| 4894 |
unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits(); |
0 |
4894 |
unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits(); |
0 |
| 4895 |
Hi = DAG.getZeroExtendInReg(Hi, dl, |
0 |
4895 |
Hi = DAG.getZeroExtendInReg(Hi, dl, |
0 |
| 4896 |
EVT::getIntegerVT(*DAG.getContext(), |
0 |
4896 |
EVT::getIntegerVT(*DAG.getContext(), |
0 |
| 4897 |
ExcessBits)); |
--- |
4897 |
ExcessBits)); |
--- |
| 4898 |
} |
--- |
4898 |
} |
--- |
| 4899 |
} |
0 |
4899 |
} |
0 |
| 4900 |
|
--- |
4900 |
|
--- |
| 4901 |
void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N, |
0 |
4901 |
void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N, |
0 |
| 4902 |
SDValue &Lo, SDValue &Hi) { |
--- |
4902 |
SDValue &Lo, SDValue &Hi) { |
--- |
| 4903 |
SDLoc dl(N); |
0 |
4903 |
SDLoc dl(N); |
0 |
| 4904 |
EVT VT = cast(N)->getMemoryVT(); |
0 |
4904 |
EVT VT = cast(N)->getMemoryVT(); |
0 |
| 4905 |
SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other); |
0 |
4905 |
SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other); |
0 |
| 4906 |
SDValue Zero = DAG.getConstant(0, dl, VT); |
0 |
4906 |
SDValue Zero = DAG.getConstant(0, dl, VT); |
0 |
| 4907 |
SDValue Swap = DAG.getAtomicCmpSwap( |
0 |
4907 |
SDValue Swap = DAG.getAtomicCmpSwap( |
0 |
| 4908 |
ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, |
--- |
4908 |
ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, |
--- |
| 4909 |
cast(N)->getMemoryVT(), VTs, N->getOperand(0), |
0 |
4909 |
cast(N)->getMemoryVT(), VTs, N->getOperand(0), |
0 |
| 4910 |
N->getOperand(1), Zero, Zero, cast(N)->getMemOperand()); |
0 |
4910 |
N->getOperand(1), Zero, Zero, cast(N)->getMemOperand()); |
0 |
| 4911 |
|
--- |
4911 |
|
--- |
| 4912 |
ReplaceValueWith(SDValue(N, 0), Swap.getValue(0)); |
0 |
4912 |
ReplaceValueWith(SDValue(N, 0), Swap.getValue(0)); |
0 |
| 4913 |
ReplaceValueWith(SDValue(N, 1), Swap.getValue(2)); |
0 |
4913 |
ReplaceValueWith(SDValue(N, 1), Swap.getValue(2)); |
0 |
| 4914 |
} |
0 |
4914 |
} |
0 |
| 4915 |
|
--- |
4915 |
|
--- |
| 4916 |
void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(SDNode *N, |
0 |
4916 |
void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(SDNode *N, |
0 |
| 4917 |
SDValue &Lo, SDValue &Hi) { |
--- |
4917 |
SDValue &Lo, SDValue &Hi) { |
--- |
| 4918 |
// TODO For VECREDUCE_(AND|OR|XOR) we could split the vector and calculate |
--- |
4918 |
// TODO For VECREDUCE_(AND|OR|XOR) we could split the vector and calculate |
--- |
| 4919 |
// both halves independently. |
--- |
4919 |
// both halves independently. |
--- |
| 4920 |
SDValue Res = TLI.expandVecReduce(N, DAG); |
0 |
4920 |
SDValue Res = TLI.expandVecReduce(N, DAG); |
0 |
| 4921 |
SplitInteger(Res, Lo, Hi); |
0 |
4921 |
SplitInteger(Res, Lo, Hi); |
0 |
| 4922 |
} |
0 |
4922 |
} |
0 |
| 4923 |
|
--- |
4923 |
|
--- |
| 4924 |
void DAGTypeLegalizer::ExpandIntRes_Rotate(SDNode *N, |
0 |
4924 |
void DAGTypeLegalizer::ExpandIntRes_Rotate(SDNode *N, |
0 |
| 4925 |
SDValue &Lo, SDValue &Hi) { |
--- |
4925 |
SDValue &Lo, SDValue &Hi) { |
--- |
| 4926 |
// Delegate to funnel-shift expansion. |
--- |
4926 |
// Delegate to funnel-shift expansion. |
--- |
| 4927 |
SDLoc DL(N); |
0 |
4927 |
SDLoc DL(N); |
0 |
| 4928 |
unsigned Opcode = N->getOpcode() == ISD::ROTL ? ISD::FSHL : ISD::FSHR; |
0 |
4928 |
unsigned Opcode = N->getOpcode() == ISD::ROTL ? ISD::FSHL : ISD::FSHR; |
0 |
| 4929 |
SDValue Res = DAG.getNode(Opcode, DL, N->getValueType(0), N->getOperand(0), |
0 |
4929 |
SDValue Res = DAG.getNode(Opcode, DL, N->getValueType(0), N->getOperand(0), |
0 |
| 4930 |
N->getOperand(0), N->getOperand(1)); |
0 |
4930 |
N->getOperand(0), N->getOperand(1)); |
0 |
| 4931 |
SplitInteger(Res, Lo, Hi); |
0 |
4931 |
SplitInteger(Res, Lo, Hi); |
0 |
| 4932 |
} |
0 |
4932 |
} |
0 |
| 4933 |
|
--- |
4933 |
|
--- |
| 4934 |
void DAGTypeLegalizer::ExpandIntRes_FunnelShift(SDNode *N, SDValue &Lo, |
0 |
4934 |
void DAGTypeLegalizer::ExpandIntRes_FunnelShift(SDNode *N, SDValue &Lo, |
0 |
| 4935 |
SDValue &Hi) { |
--- |
4935 |
SDValue &Hi) { |
--- |
| 4936 |
// Values numbered from least significant to most significant. |
--- |
4936 |
// Values numbered from least significant to most significant. |
--- |
| 4937 |
SDValue In1, In2, In3, In4; |
0 |
4937 |
SDValue In1, In2, In3, In4; |
0 |
| 4938 |
GetExpandedInteger(N->getOperand(0), In3, In4); |
0 |
4938 |
GetExpandedInteger(N->getOperand(0), In3, In4); |
0 |
| 4939 |
GetExpandedInteger(N->getOperand(1), In1, In2); |
0 |
4939 |
GetExpandedInteger(N->getOperand(1), In1, In2); |
0 |
| 4940 |
EVT HalfVT = In1.getValueType(); |
0 |
4940 |
EVT HalfVT = In1.getValueType(); |
0 |
| 4941 |
|
--- |
4941 |
|
--- |
| 4942 |
SDLoc DL(N); |
0 |
4942 |
SDLoc DL(N); |
0 |
| 4943 |
unsigned Opc = N->getOpcode(); |
0 |
4943 |
unsigned Opc = N->getOpcode(); |
0 |
| 4944 |
SDValue ShAmt = N->getOperand(2); |
0 |
4944 |
SDValue ShAmt = N->getOperand(2); |
0 |
| 4945 |
EVT ShAmtVT = ShAmt.getValueType(); |
0 |
4945 |
EVT ShAmtVT = ShAmt.getValueType(); |
0 |
| 4946 |
EVT ShAmtCCVT = getSetCCResultType(ShAmtVT); |
0 |
4946 |
EVT ShAmtCCVT = getSetCCResultType(ShAmtVT); |
0 |
| 4947 |
|
--- |
4947 |
|
--- |
| 4948 |
// If the shift amount is at least half the bitwidth, swap the inputs. |
--- |
4948 |
// If the shift amount is at least half the bitwidth, swap the inputs. |
--- |
| 4949 |
unsigned HalfVTBits = HalfVT.getScalarSizeInBits(); |
0 |
4949 |
unsigned HalfVTBits = HalfVT.getScalarSizeInBits(); |
0 |
| 4950 |
SDValue AndNode = DAG.getNode(ISD::AND, DL, ShAmtVT, ShAmt, |
0 |
4950 |
SDValue AndNode = DAG.getNode(ISD::AND, DL, ShAmtVT, ShAmt, |
0 |
| 4951 |
DAG.getConstant(HalfVTBits, DL, ShAmtVT)); |
0 |
4951 |
DAG.getConstant(HalfVTBits, DL, ShAmtVT)); |
0 |
| 4952 |
SDValue Cond = |
--- |
4952 |
SDValue Cond = |
--- |
| 4953 |
DAG.getSetCC(DL, ShAmtCCVT, AndNode, DAG.getConstant(0, DL, ShAmtVT), |
0 |
4953 |
DAG.getSetCC(DL, ShAmtCCVT, AndNode, DAG.getConstant(0, DL, ShAmtVT), |
0 |
| 4954 |
Opc == ISD::FSHL ? ISD::SETNE : ISD::SETEQ); |
--- |
4954 |
Opc == ISD::FSHL ? ISD::SETNE : ISD::SETEQ); |
--- |
| 4955 |
|
--- |
4955 |
|
--- |
| 4956 |
// Expand to a pair of funnel shifts. |
--- |
4956 |
// Expand to a pair of funnel shifts. |
--- |
| 4957 |
EVT NewShAmtVT = TLI.getShiftAmountTy(HalfVT, DAG.getDataLayout()); |
0 |
4957 |
EVT NewShAmtVT = TLI.getShiftAmountTy(HalfVT, DAG.getDataLayout()); |
0 |
| 4958 |
SDValue NewShAmt = DAG.getAnyExtOrTrunc(ShAmt, DL, NewShAmtVT); |
0 |
4958 |
SDValue NewShAmt = DAG.getAnyExtOrTrunc(ShAmt, DL, NewShAmtVT); |
0 |
| 4959 |
|
--- |
4959 |
|
--- |
| 4960 |
SDValue Select1 = DAG.getNode(ISD::SELECT, DL, HalfVT, Cond, In1, In2); |
0 |
4960 |
SDValue Select1 = DAG.getNode(ISD::SELECT, DL, HalfVT, Cond, In1, In2); |
0 |
| 4961 |
SDValue Select2 = DAG.getNode(ISD::SELECT, DL, HalfVT, Cond, In2, In3); |
0 |
4961 |
SDValue Select2 = DAG.getNode(ISD::SELECT, DL, HalfVT, Cond, In2, In3); |
0 |
| 4962 |
SDValue Select3 = DAG.getNode(ISD::SELECT, DL, HalfVT, Cond, In3, In4); |
0 |
4962 |
SDValue Select3 = DAG.getNode(ISD::SELECT, DL, HalfVT, Cond, In3, In4); |
0 |
| 4963 |
Lo = DAG.getNode(Opc, DL, HalfVT, Select2, Select1, NewShAmt); |
0 |
4963 |
Lo = DAG.getNode(Opc, DL, HalfVT, Select2, Select1, NewShAmt); |
0 |
| 4964 |
Hi = DAG.getNode(Opc, DL, HalfVT, Select3, Select2, NewShAmt); |
0 |
4964 |
Hi = DAG.getNode(Opc, DL, HalfVT, Select3, Select2, NewShAmt); |
0 |
| 4965 |
} |
0 |
4965 |
} |
0 |
| 4966 |
|
--- |
4966 |
|
--- |
| 4967 |
void DAGTypeLegalizer::ExpandIntRes_VSCALE(SDNode *N, SDValue &Lo, |
0 |
4967 |
void DAGTypeLegalizer::ExpandIntRes_VSCALE(SDNode *N, SDValue &Lo, |
0 |
| 4968 |
SDValue &Hi) { |
--- |
4968 |
SDValue &Hi) { |
--- |
| 4969 |
EVT VT = N->getValueType(0); |
0 |
4969 |
EVT VT = N->getValueType(0); |
0 |
| 4970 |
EVT HalfVT = |
--- |
4970 |
EVT HalfVT = |
--- |
| 4971 |
EVT::getIntegerVT(*DAG.getContext(), N->getValueSizeInBits(0) / 2); |
0 |
4971 |
EVT::getIntegerVT(*DAG.getContext(), N->getValueSizeInBits(0) / 2); |
0 |
| 4972 |
SDLoc dl(N); |
0 |
4972 |
SDLoc dl(N); |
0 |
| 4973 |
|
--- |
4973 |
|
--- |
| 4974 |
// We assume VSCALE(1) fits into a legal integer. |
--- |
4974 |
// We assume VSCALE(1) fits into a legal integer. |
--- |
| 4975 |
APInt One(HalfVT.getSizeInBits(), 1); |
0 |
4975 |
APInt One(HalfVT.getSizeInBits(), 1); |
0 |
| 4976 |
SDValue VScaleBase = DAG.getVScale(dl, HalfVT, One); |
0 |
4976 |
SDValue VScaleBase = DAG.getVScale(dl, HalfVT, One); |
0 |
| 4977 |
VScaleBase = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, VScaleBase); |
0 |
4977 |
VScaleBase = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, VScaleBase); |
0 |
| 4978 |
SDValue Res = DAG.getNode(ISD::MUL, dl, VT, VScaleBase, N->getOperand(0)); |
0 |
4978 |
SDValue Res = DAG.getNode(ISD::MUL, dl, VT, VScaleBase, N->getOperand(0)); |
0 |
| 4979 |
SplitInteger(Res, Lo, Hi); |
0 |
4979 |
SplitInteger(Res, Lo, Hi); |
0 |
| 4980 |
} |
0 |
4980 |
} |
0 |
| 4981 |
|
--- |
4981 |
|
--- |
| 4982 |
//===----------------------------------------------------------------------===// |
--- |
4982 |
//===----------------------------------------------------------------------===// |
--- |
| 4983 |
// Integer Operand Expansion |
--- |
4983 |
// Integer Operand Expansion |
--- |
| 4984 |
//===----------------------------------------------------------------------===// |
--- |
4984 |
//===----------------------------------------------------------------------===// |
--- |
| 4985 |
|
--- |
4985 |
|
--- |
| 4986 |
/// ExpandIntegerOperand - This method is called when the specified operand of |
--- |
4986 |
/// ExpandIntegerOperand - This method is called when the specified operand of |
--- |
| 4987 |
/// the specified node is found to need expansion. At this point, all of the |
--- |
4987 |
/// the specified node is found to need expansion. At this point, all of the |
--- |
| 4988 |
/// result types of the node are known to be legal, but other operands of the |
--- |
4988 |
/// result types of the node are known to be legal, but other operands of the |
--- |
| 4989 |
/// node may need promotion or expansion as well as the specified one. |
--- |
4989 |
/// node may need promotion or expansion as well as the specified one. |
--- |
| 4990 |
bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) { |
0 |
4990 |
bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) { |
0 |
| 4991 |
LLVM_DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG); |
0 |
4991 |
LLVM_DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG); |
0 |
| 4992 |
dbgs() << "\n"); |
--- |
4992 |
dbgs() << "\n"); |
--- |
| 4993 |
SDValue Res = SDValue(); |
0 |
4993 |
SDValue Res = SDValue(); |
0 |
| 4994 |
|
--- |
4994 |
|
--- |
| 4995 |
if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) |
0 |
4995 |
if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) |
0 |
| 4996 |
return false; |
0 |
4996 |
return false; |
0 |
| 4997 |
|
--- |
4997 |
|
--- |
| 4998 |
switch (N->getOpcode()) { |
0 |
4998 |
switch (N->getOpcode()) { |
0 |
| 4999 |
default: |
0 |
4999 |
default: |
0 |
| 5000 |
#ifndef NDEBUG |
--- |
5000 |
#ifndef NDEBUG |
--- |
| 5001 |
dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": "; |
0 |
5001 |
dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": "; |
0 |
| 5002 |
N->dump(&DAG); dbgs() << "\n"; |
0 |
5002 |
N->dump(&DAG); dbgs() << "\n"; |
0 |
| 5003 |
#endif |
--- |
5003 |
#endif |
--- |
| 5004 |
report_fatal_error("Do not know how to expand this operator's operand!"); |
0 |
5004 |
report_fatal_error("Do not know how to expand this operator's operand!"); |
0 |
| 5005 |
|
--- |
5005 |
|
--- |
| 5006 |
case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break; |
0 |
5006 |
case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break; |
0 |
| 5007 |
case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break; |
0 |
5007 |
case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break; |
0 |
| 5008 |
case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break; |
0 |
5008 |
case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break; |
0 |
| 5009 |
case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break; |
0 |
5009 |
case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break; |
0 |
| 5010 |
case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break; |
0 |
5010 |
case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break; |
0 |
| 5011 |
case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break; |
0 |
5011 |
case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break; |
0 |
| 5012 |
case ISD::SPLAT_VECTOR: Res = ExpandIntOp_SPLAT_VECTOR(N); break; |
0 |
5012 |
case ISD::SPLAT_VECTOR: Res = ExpandIntOp_SPLAT_VECTOR(N); break; |
0 |
| 5013 |
case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break; |
0 |
5013 |
case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break; |
0 |
| 5014 |
case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break; |
0 |
5014 |
case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break; |
0 |
| 5015 |
case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break; |
0 |
5015 |
case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break; |
0 |
| 5016 |
case ISD::STRICT_SINT_TO_FP: |
0 |
5016 |
case ISD::STRICT_SINT_TO_FP: |
0 |
| 5017 |
case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break; |
0 |
5017 |
case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break; |
0 |
| 5018 |
case ISD::STORE: Res = ExpandIntOp_STORE(cast(N), OpNo); break; |
0 |
5018 |
case ISD::STORE: Res = ExpandIntOp_STORE(cast(N), OpNo); break; |
0 |
| 5019 |
case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break; |
0 |
5019 |
case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break; |
0 |
| 5020 |
case ISD::STRICT_UINT_TO_FP: |
0 |
5020 |
case ISD::STRICT_UINT_TO_FP: |
0 |
| 5021 |
case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break; |
0 |
5021 |
case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break; |
0 |
| 5022 |
|
--- |
5022 |
|
--- |
| 5023 |
case ISD::SHL: |
0 |
5023 |
case ISD::SHL: |
0 |
| 5024 |
case ISD::SRA: |
--- |
5024 |
case ISD::SRA: |
--- |
| 5025 |
case ISD::SRL: |
--- |
5025 |
case ISD::SRL: |
--- |
| 5026 |
case ISD::ROTL: |
--- |
5026 |
case ISD::ROTL: |
--- |
| 5027 |
case ISD::ROTR: Res = ExpandIntOp_Shift(N); break; |
0 |
5027 |
case ISD::ROTR: Res = ExpandIntOp_Shift(N); break; |
0 |
| 5028 |
case ISD::RETURNADDR: |
0 |
5028 |
case ISD::RETURNADDR: |
0 |
| 5029 |
case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break; |
0 |
5029 |
case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break; |
0 |
| 5030 |
|
--- |
5030 |
|
--- |
| 5031 |
case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break; |
0 |
5031 |
case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break; |
0 |
| 5032 |
case ISD::STACKMAP: |
0 |
5032 |
case ISD::STACKMAP: |
0 |
| 5033 |
Res = ExpandIntOp_STACKMAP(N, OpNo); |
0 |
5033 |
Res = ExpandIntOp_STACKMAP(N, OpNo); |
0 |
| 5034 |
break; |
0 |
5034 |
break; |
0 |
| 5035 |
case ISD::PATCHPOINT: |
0 |
5035 |
case ISD::PATCHPOINT: |
0 |
| 5036 |
Res = ExpandIntOp_PATCHPOINT(N, OpNo); |
0 |
5036 |
Res = ExpandIntOp_PATCHPOINT(N, OpNo); |
0 |
| 5037 |
break; |
0 |
5037 |
break; |
0 |
| 5038 |
case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: |
0 |
5038 |
case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: |
0 |
| 5039 |
case ISD::EXPERIMENTAL_VP_STRIDED_STORE: |
--- |
5039 |
case ISD::EXPERIMENTAL_VP_STRIDED_STORE: |
--- |
| 5040 |
Res = ExpandIntOp_VP_STRIDED(N, OpNo); |
0 |
5040 |
Res = ExpandIntOp_VP_STRIDED(N, OpNo); |
0 |
| 5041 |
break; |
0 |
5041 |
break; |
0 |
| 5042 |
} |
--- |
5042 |
} |
--- |
| 5043 |
|
--- |
5043 |
|
--- |
| 5044 |
// If the result is null, the sub-method took care of registering results etc. |
--- |
5044 |
// If the result is null, the sub-method took care of registering results etc. |
--- |
| 5045 |
if (!Res.getNode()) return false; |
0 |
5045 |
if (!Res.getNode()) return false; |
0 |
| 5046 |
|
--- |
5046 |
|
--- |
| 5047 |
// If the result is N, the sub-method updated N in place. Tell the legalizer |
--- |
5047 |
// If the result is N, the sub-method updated N in place. Tell the legalizer |
--- |
| 5048 |
// core about this. |
--- |
5048 |
// core about this. |
--- |
| 5049 |
if (Res.getNode() == N) |
0 |
5049 |
if (Res.getNode() == N) |
0 |
| 5050 |
return true; |
0 |
5050 |
return true; |
0 |
| 5051 |
|
--- |
5051 |
|
--- |
| 5052 |
assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && |
0 |
5052 |
assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && |
0 |
| 5053 |
"Invalid operand expansion"); |
--- |
5053 |
"Invalid operand expansion"); |
--- |
| 5054 |
|
--- |
5054 |
|
--- |
| 5055 |
ReplaceValueWith(SDValue(N, 0), Res); |
0 |
5055 |
ReplaceValueWith(SDValue(N, 0), Res); |
0 |
| 5056 |
return false; |
0 |
5056 |
return false; |
0 |
| 5057 |
} |
--- |
5057 |
} |
--- |
| 5058 |
|
--- |
5058 |
|
--- |
| 5059 |
/// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code |
--- |
5059 |
/// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code |
--- |
| 5060 |
/// is shared among BR_CC, SELECT_CC, and SETCC handlers. |
--- |
5060 |
/// is shared among BR_CC, SELECT_CC, and SETCC handlers. |
--- |
| 5061 |
void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS, |
0 |
5061 |
void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS, |
0 |
| 5062 |
SDValue &NewRHS, |
--- |
5062 |
SDValue &NewRHS, |
--- |
| 5063 |
ISD::CondCode &CCCode, |
--- |
5063 |
ISD::CondCode &CCCode, |
--- |
| 5064 |
const SDLoc &dl) { |
--- |
5064 |
const SDLoc &dl) { |
--- |
| 5065 |
SDValue LHSLo, LHSHi, RHSLo, RHSHi; |
0 |
5065 |
SDValue LHSLo, LHSHi, RHSLo, RHSHi; |
0 |
| 5066 |
GetExpandedInteger(NewLHS, LHSLo, LHSHi); |
0 |
5066 |
GetExpandedInteger(NewLHS, LHSLo, LHSHi); |
0 |
| 5067 |
GetExpandedInteger(NewRHS, RHSLo, RHSHi); |
0 |
5067 |
GetExpandedInteger(NewRHS, RHSLo, RHSHi); |
0 |
| 5068 |
|
--- |
5068 |
|
--- |
| 5069 |
if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) { |
0 |
5069 |
if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) { |
0 |
| 5070 |
if (RHSLo == RHSHi) { |
0 |
5070 |
if (RHSLo == RHSHi) { |
0 |
| 5071 |
if (ConstantSDNode *RHSCST = dyn_cast(RHSLo)) { |
0 |
5071 |
if (ConstantSDNode *RHSCST = dyn_cast(RHSLo)) { |
0 |
| 5072 |
if (RHSCST->isAllOnes()) { |
0 |
5072 |
if (RHSCST->isAllOnes()) { |
0 |
| 5073 |
// Equality comparison to -1. |
--- |
5073 |
// Equality comparison to -1. |
--- |
| 5074 |
NewLHS = DAG.getNode(ISD::AND, dl, |
0 |
5074 |
NewLHS = DAG.getNode(ISD::AND, dl, |
0 |
| 5075 |
LHSLo.getValueType(), LHSLo, LHSHi); |
--- |
5075 |
LHSLo.getValueType(), LHSLo, LHSHi); |
--- |
| 5076 |
NewRHS = RHSLo; |
0 |
5076 |
NewRHS = RHSLo; |
0 |
| 5077 |
return; |
0 |
5077 |
return; |
0 |
| 5078 |
} |
--- |
5078 |
} |
--- |
| 5079 |
} |
--- |
5079 |
} |
--- |
| 5080 |
} |
--- |
5080 |
} |
--- |
| 5081 |
|
--- |
5081 |
|
--- |
| 5082 |
NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo); |
0 |
5082 |
NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo); |
0 |
| 5083 |
NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi); |
0 |
5083 |
NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi); |
0 |
| 5084 |
NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS); |
0 |
5084 |
NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS); |
0 |
| 5085 |
NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType()); |
0 |
5085 |
NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType()); |
0 |
| 5086 |
return; |
0 |
5086 |
return; |
0 |
| 5087 |
} |
--- |
5087 |
} |
--- |
| 5088 |
|
--- |
5088 |
|
--- |
| 5089 |
// If this is a comparison of the sign bit, just look at the top part. |
--- |
5089 |
// If this is a comparison of the sign bit, just look at the top part. |
--- |
| 5090 |
// X > -1, x < 0 |
--- |
5090 |
// X > -1, x < 0 |
--- |
| 5091 |
if (ConstantSDNode *CST = dyn_cast(NewRHS)) |
0 |
5091 |
if (ConstantSDNode *CST = dyn_cast(NewRHS)) |
0 |
| 5092 |
if ((CCCode == ISD::SETLT && CST->isZero()) || // X < 0 |
0 |
5092 |
if ((CCCode == ISD::SETLT && CST->isZero()) || // X < 0 |
0 |
| 5093 |
(CCCode == ISD::SETGT && CST->isAllOnes())) { // X > -1 |
0 |
5093 |
(CCCode == ISD::SETGT && CST->isAllOnes())) { // X > -1 |
0 |
| 5094 |
NewLHS = LHSHi; |
0 |
5094 |
NewLHS = LHSHi; |
0 |
| 5095 |
NewRHS = RHSHi; |
0 |
5095 |
NewRHS = RHSHi; |
0 |
| 5096 |
return; |
0 |
5096 |
return; |
0 |
| 5097 |
} |
--- |
5097 |
} |
--- |
| 5098 |
|
--- |
5098 |
|
--- |
| 5099 |
// FIXME: This generated code sucks. |
--- |
5099 |
// FIXME: This generated code sucks. |
--- |
| 5100 |
ISD::CondCode LowCC; |
--- |
5100 |
ISD::CondCode LowCC; |
--- |
| 5101 |
switch (CCCode) { |
0 |
5101 |
switch (CCCode) { |
0 |
| 5102 |
default: llvm_unreachable("Unknown integer setcc!"); |
0 |
5102 |
default: llvm_unreachable("Unknown integer setcc!"); |
0 |
| 5103 |
case ISD::SETLT: |
0 |
5103 |
case ISD::SETLT: |
0 |
| 5104 |
case ISD::SETULT: LowCC = ISD::SETULT; break; |
0 |
5104 |
case ISD::SETULT: LowCC = ISD::SETULT; break; |
0 |
| 5105 |
case ISD::SETGT: |
0 |
5105 |
case ISD::SETGT: |
0 |
| 5106 |
case ISD::SETUGT: LowCC = ISD::SETUGT; break; |
0 |
5106 |
case ISD::SETUGT: LowCC = ISD::SETUGT; break; |
0 |
| 5107 |
case ISD::SETLE: |
0 |
5107 |
case ISD::SETLE: |
0 |
| 5108 |
case ISD::SETULE: LowCC = ISD::SETULE; break; |
0 |
5108 |
case ISD::SETULE: LowCC = ISD::SETULE; break; |
0 |
| 5109 |
case ISD::SETGE: |
0 |
5109 |
case ISD::SETGE: |
0 |
| 5110 |
case ISD::SETUGE: LowCC = ISD::SETUGE; break; |
0 |
5110 |
case ISD::SETUGE: LowCC = ISD::SETUGE; break; |
0 |
| 5111 |
} |
--- |
5111 |
} |
--- |
| 5112 |
|
--- |
5112 |
|
--- |
| 5113 |
// LoCmp = lo(op1) < lo(op2) // Always unsigned comparison |
--- |
5113 |
// LoCmp = lo(op1) < lo(op2) // Always unsigned comparison |
--- |
| 5114 |
// HiCmp = hi(op1) < hi(op2) // Signedness depends on operands |
--- |
5114 |
// HiCmp = hi(op1) < hi(op2) // Signedness depends on operands |
--- |
| 5115 |
// dest = hi(op1) == hi(op2) ? LoCmp : HiCmp; |
--- |
5115 |
// dest = hi(op1) == hi(op2) ? LoCmp : HiCmp; |
--- |
| 5116 |
|
--- |
5116 |
|
--- |
| 5117 |
// NOTE: on targets without efficient SELECT of bools, we can always use |
--- |
5117 |
// NOTE: on targets without efficient SELECT of bools, we can always use |
--- |
| 5118 |
// this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3) |
--- |
5118 |
// this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3) |
--- |
| 5119 |
TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true, |
--- |
5119 |
TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true, |
--- |
| 5120 |
nullptr); |
0 |
5120 |
nullptr); |
0 |
| 5121 |
SDValue LoCmp, HiCmp; |
0 |
5121 |
SDValue LoCmp, HiCmp; |
0 |
| 5122 |
if (TLI.isTypeLegal(LHSLo.getValueType()) && |
0 |
5122 |
if (TLI.isTypeLegal(LHSLo.getValueType()) && |
0 |
| 5123 |
TLI.isTypeLegal(RHSLo.getValueType())) |
0 |
5123 |
TLI.isTypeLegal(RHSLo.getValueType())) |
0 |
| 5124 |
LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo, |
0 |
5124 |
LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo, |
0 |
| 5125 |
RHSLo, LowCC, false, DagCombineInfo, dl); |
--- |
5125 |
RHSLo, LowCC, false, DagCombineInfo, dl); |
--- |
| 5126 |
if (!LoCmp.getNode()) |
0 |
5126 |
if (!LoCmp.getNode()) |
0 |
| 5127 |
LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo, |
0 |
5127 |
LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo, |
0 |
| 5128 |
RHSLo, LowCC); |
--- |
5128 |
RHSLo, LowCC); |
--- |
| 5129 |
if (TLI.isTypeLegal(LHSHi.getValueType()) && |
0 |
5129 |
if (TLI.isTypeLegal(LHSHi.getValueType()) && |
0 |
| 5130 |
TLI.isTypeLegal(RHSHi.getValueType())) |
0 |
5130 |
TLI.isTypeLegal(RHSHi.getValueType())) |
0 |
| 5131 |
HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi, |
0 |
5131 |
HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi, |
0 |
| 5132 |
RHSHi, CCCode, false, DagCombineInfo, dl); |
--- |
5132 |
RHSHi, CCCode, false, DagCombineInfo, dl); |
--- |
| 5133 |
if (!HiCmp.getNode()) |
0 |
5133 |
if (!HiCmp.getNode()) |
0 |
| 5134 |
HiCmp = |
0 |
5134 |
HiCmp = |
0 |
| 5135 |
DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()), |
0 |
5135 |
DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()), |
0 |
| 5136 |
LHSHi, RHSHi, DAG.getCondCode(CCCode)); |
0 |
5136 |
LHSHi, RHSHi, DAG.getCondCode(CCCode)); |
0 |
| 5137 |
|
--- |
5137 |
|
--- |
| 5138 |
ConstantSDNode *LoCmpC = dyn_cast(LoCmp.getNode()); |
0 |
5138 |
ConstantSDNode *LoCmpC = dyn_cast(LoCmp.getNode()); |
0 |
| 5139 |
ConstantSDNode *HiCmpC = dyn_cast(HiCmp.getNode()); |
0 |
5139 |
ConstantSDNode *HiCmpC = dyn_cast(HiCmp.getNode()); |
0 |
| 5140 |
|
--- |
5140 |
|
--- |
| 5141 |
bool EqAllowed = ISD::isTrueWhenEqual(CCCode); |
0 |
5141 |
bool EqAllowed = ISD::isTrueWhenEqual(CCCode); |
0 |
| 5142 |
|
--- |
5142 |
|
--- |
| 5143 |
// FIXME: Is the HiCmpC->isOne() here correct for |
--- |
5143 |
// FIXME: Is the HiCmpC->isOne() here correct for |
--- |
| 5144 |
// ZeroOrNegativeOneBooleanContent. |
--- |
5144 |
// ZeroOrNegativeOneBooleanContent. |
--- |
| 5145 |
if ((EqAllowed && (HiCmpC && HiCmpC->isZero())) || |
0 |
5145 |
if ((EqAllowed && (HiCmpC && HiCmpC->isZero())) || |
0 |
| 5146 |
(!EqAllowed && |
0 |
5146 |
(!EqAllowed && |
0 |
| 5147 |
((HiCmpC && HiCmpC->isOne()) || (LoCmpC && LoCmpC->isZero())))) { |
0 |
5147 |
((HiCmpC && HiCmpC->isOne()) || (LoCmpC && LoCmpC->isZero())))) { |
0 |
| 5148 |
// For LE / GE, if high part is known false, ignore the low part. |
--- |
5148 |
// For LE / GE, if high part is known false, ignore the low part. |
--- |
| 5149 |
// For LT / GT: if low part is known false, return the high part. |
--- |
5149 |
// For LT / GT: if low part is known false, return the high part. |
--- |
| 5150 |
// if high part is known true, ignore the low part. |
--- |
5150 |
// if high part is known true, ignore the low part. |
--- |
| 5151 |
NewLHS = HiCmp; |
0 |
5151 |
NewLHS = HiCmp; |
0 |
| 5152 |
NewRHS = SDValue(); |
0 |
5152 |
NewRHS = SDValue(); |
0 |
| 5153 |
return; |
0 |
5153 |
return; |
0 |
| 5154 |
} |
--- |
5154 |
} |
--- |
| 5155 |
|
--- |
5155 |
|
--- |
| 5156 |
if (LHSHi == RHSHi) { |
0 |
5156 |
if (LHSHi == RHSHi) { |
0 |
| 5157 |
// Comparing the low bits is enough. |
--- |
5157 |
// Comparing the low bits is enough. |
--- |
| 5158 |
NewLHS = LoCmp; |
0 |
5158 |
NewLHS = LoCmp; |
0 |
| 5159 |
NewRHS = SDValue(); |
0 |
5159 |
NewRHS = SDValue(); |
0 |
| 5160 |
return; |
0 |
5160 |
return; |
0 |
| 5161 |
} |
--- |
5161 |
} |
--- |
| 5162 |
|
--- |
5162 |
|
--- |
| 5163 |
// Lower with SETCCCARRY if the target supports it. |
--- |
5163 |
// Lower with SETCCCARRY if the target supports it. |
--- |
| 5164 |
EVT HiVT = LHSHi.getValueType(); |
0 |
5164 |
EVT HiVT = LHSHi.getValueType(); |
0 |
| 5165 |
EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT); |
0 |
5165 |
EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT); |
0 |
| 5166 |
bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT); |
0 |
5166 |
bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT); |
0 |
| 5167 |
|
--- |
5167 |
|
--- |
| 5168 |
// FIXME: Make all targets support this, then remove the other lowering. |
--- |
5168 |
// FIXME: Make all targets support this, then remove the other lowering. |
--- |
| 5169 |
if (HasSETCCCARRY) { |
0 |
5169 |
if (HasSETCCCARRY) { |
0 |
| 5170 |
// SETCCCARRY can detect < and >= directly. For > and <=, flip |
--- |
5170 |
// SETCCCARRY can detect < and >= directly. For > and <=, flip |
--- |
| 5171 |
// operands and condition code. |
--- |
5171 |
// operands and condition code. |
--- |
| 5172 |
bool FlipOperands = false; |
0 |
5172 |
bool FlipOperands = false; |
0 |
| 5173 |
switch (CCCode) { |
0 |
5173 |
switch (CCCode) { |
0 |
| 5174 |
case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break; |
0 |
5174 |
case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break; |
0 |
| 5175 |
case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break; |
0 |
5175 |
case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break; |
0 |
| 5176 |
case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break; |
0 |
5176 |
case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break; |
0 |
| 5177 |
case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break; |
0 |
5177 |
case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break; |
0 |
| 5178 |
default: break; |
0 |
5178 |
default: break; |
0 |
| 5179 |
} |
--- |
5179 |
} |
--- |
| 5180 |
if (FlipOperands) { |
0 |
5180 |
if (FlipOperands) { |
0 |
| 5181 |
std::swap(LHSLo, RHSLo); |
0 |
5181 |
std::swap(LHSLo, RHSLo); |
0 |
| 5182 |
std::swap(LHSHi, RHSHi); |
0 |
5182 |
std::swap(LHSHi, RHSHi); |
0 |
| 5183 |
} |
--- |
5183 |
} |
--- |
| 5184 |
// Perform a wide subtraction, feeding the carry from the low part into |
--- |
5184 |
// Perform a wide subtraction, feeding the carry from the low part into |
--- |
| 5185 |
// SETCCCARRY. The SETCCCARRY operation is essentially looking at the high |
--- |
5185 |
// SETCCCARRY. The SETCCCARRY operation is essentially looking at the high |
--- |
| 5186 |
// part of the result of LHS - RHS. It is negative iff LHS < RHS. It is |
--- |
5186 |
// part of the result of LHS - RHS. It is negative iff LHS < RHS. It is |
--- |
| 5187 |
// zero or positive iff LHS >= RHS. |
--- |
5187 |
// zero or positive iff LHS >= RHS. |
--- |
| 5188 |
EVT LoVT = LHSLo.getValueType(); |
0 |
5188 |
EVT LoVT = LHSLo.getValueType(); |
0 |
| 5189 |
SDVTList VTList = DAG.getVTList(LoVT, getSetCCResultType(LoVT)); |
0 |
5189 |
SDVTList VTList = DAG.getVTList(LoVT, getSetCCResultType(LoVT)); |
0 |
| 5190 |
SDValue LowCmp = DAG.getNode(ISD::USUBO, dl, VTList, LHSLo, RHSLo); |
0 |
5190 |
SDValue LowCmp = DAG.getNode(ISD::USUBO, dl, VTList, LHSLo, RHSLo); |
0 |
| 5191 |
SDValue Res = DAG.getNode(ISD::SETCCCARRY, dl, getSetCCResultType(HiVT), |
0 |
5191 |
SDValue Res = DAG.getNode(ISD::SETCCCARRY, dl, getSetCCResultType(HiVT), |
0 |
| 5192 |
LHSHi, RHSHi, LowCmp.getValue(1), |
--- |
5192 |
LHSHi, RHSHi, LowCmp.getValue(1), |
--- |
| 5193 |
DAG.getCondCode(CCCode)); |
0 |
5193 |
DAG.getCondCode(CCCode)); |
0 |
| 5194 |
NewLHS = Res; |
0 |
5194 |
NewLHS = Res; |
0 |
| 5195 |
NewRHS = SDValue(); |
0 |
5195 |
NewRHS = SDValue(); |
0 |
| 5196 |
return; |
0 |
5196 |
return; |
0 |
| 5197 |
} |
--- |
5197 |
} |
--- |
| 5198 |
|
--- |
5198 |
|
--- |
| 5199 |
NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ, |
0 |
5199 |
NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ, |
0 |
| 5200 |
false, DagCombineInfo, dl); |
--- |
5200 |
false, DagCombineInfo, dl); |
--- |
| 5201 |
if (!NewLHS.getNode()) |
0 |
5201 |
if (!NewLHS.getNode()) |
0 |
| 5202 |
NewLHS = |
0 |
5202 |
NewLHS = |
0 |
| 5203 |
DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ); |
0 |
5203 |
DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ); |
0 |
| 5204 |
NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp); |
0 |
5204 |
NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp); |
0 |
| 5205 |
NewRHS = SDValue(); |
0 |
5205 |
NewRHS = SDValue(); |
0 |
| 5206 |
} |
--- |
5206 |
} |
--- |
| 5207 |
|
--- |
5207 |
|
--- |
| 5208 |
SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) { |
0 |
5208 |
SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) { |
0 |
| 5209 |
SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); |
0 |
5209 |
SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); |
0 |
| 5210 |
ISD::CondCode CCCode = cast(N->getOperand(1))->get(); |
0 |
5210 |
ISD::CondCode CCCode = cast(N->getOperand(1))->get(); |
0 |
| 5211 |
IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); |
0 |
5211 |
IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); |
0 |
| 5212 |
|
--- |
5212 |
|
--- |
| 5213 |
// If ExpandSetCCOperands returned a scalar, we need to compare the result |
--- |
5213 |
// If ExpandSetCCOperands returned a scalar, we need to compare the result |
--- |
| 5214 |
// against zero to select between true and false values. |
--- |
5214 |
// against zero to select between true and false values. |
--- |
| 5215 |
if (!NewRHS.getNode()) { |
0 |
5215 |
if (!NewRHS.getNode()) { |
0 |
| 5216 |
NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); |
0 |
5216 |
NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); |
0 |
| 5217 |
CCCode = ISD::SETNE; |
0 |
5217 |
CCCode = ISD::SETNE; |
0 |
| 5218 |
} |
--- |
5218 |
} |
--- |
| 5219 |
|
--- |
5219 |
|
--- |
| 5220 |
// Update N to have the operands specified. |
--- |
5220 |
// Update N to have the operands specified. |
--- |
| 5221 |
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), |
0 |
5221 |
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), |
0 |
| 5222 |
DAG.getCondCode(CCCode), NewLHS, NewRHS, |
0 |
5222 |
DAG.getCondCode(CCCode), NewLHS, NewRHS, |
0 |
| 5223 |
N->getOperand(4)), 0); |
0 |
5223 |
N->getOperand(4)), 0); |
0 |
| 5224 |
} |
--- |
5224 |
} |
--- |
| 5225 |
|
--- |
5225 |
|
--- |
| 5226 |
SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) { |
0 |
5226 |
SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) { |
0 |
| 5227 |
SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); |
0 |
5227 |
SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); |
0 |
| 5228 |
ISD::CondCode CCCode = cast(N->getOperand(4))->get(); |
0 |
5228 |
ISD::CondCode CCCode = cast(N->getOperand(4))->get(); |
0 |
| 5229 |
IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); |
0 |
5229 |
IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); |
0 |
| 5230 |
|
--- |
5230 |
|
--- |
| 5231 |
// If ExpandSetCCOperands returned a scalar, we need to compare the result |
--- |
5231 |
// If ExpandSetCCOperands returned a scalar, we need to compare the result |
--- |
| 5232 |
// against zero to select between true and false values. |
--- |
5232 |
// against zero to select between true and false values. |
--- |
| 5233 |
if (!NewRHS.getNode()) { |
0 |
5233 |
if (!NewRHS.getNode()) { |
0 |
| 5234 |
NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); |
0 |
5234 |
NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); |
0 |
| 5235 |
CCCode = ISD::SETNE; |
0 |
5235 |
CCCode = ISD::SETNE; |
0 |
| 5236 |
} |
--- |
5236 |
} |
--- |
| 5237 |
|
--- |
5237 |
|
--- |
| 5238 |
// Update N to have the operands specified. |
--- |
5238 |
// Update N to have the operands specified. |
--- |
| 5239 |
return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, |
0 |
5239 |
return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, |
0 |
| 5240 |
N->getOperand(2), N->getOperand(3), |
0 |
5240 |
N->getOperand(2), N->getOperand(3), |
0 |
| 5241 |
DAG.getCondCode(CCCode)), 0); |
0 |
5241 |
DAG.getCondCode(CCCode)), 0); |
0 |
| 5242 |
} |
--- |
5242 |
} |
--- |
| 5243 |
|
--- |
5243 |
|
--- |
| 5244 |
SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) { |
0 |
5244 |
SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) { |
0 |
| 5245 |
SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); |
0 |
5245 |
SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); |
0 |
| 5246 |
ISD::CondCode CCCode = cast(N->getOperand(2))->get(); |
0 |
5246 |
ISD::CondCode CCCode = cast(N->getOperand(2))->get(); |
0 |
| 5247 |
IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); |
0 |
5247 |
IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); |
0 |
| 5248 |
|
--- |
5248 |
|
--- |
| 5249 |
// If ExpandSetCCOperands returned a scalar, use it. |
--- |
5249 |
// If ExpandSetCCOperands returned a scalar, use it. |
--- |
| 5250 |
if (!NewRHS.getNode()) { |
0 |
5250 |
if (!NewRHS.getNode()) { |
0 |
| 5251 |
assert(NewLHS.getValueType() == N->getValueType(0) && |
0 |
5251 |
assert(NewLHS.getValueType() == N->getValueType(0) && |
0 |
| 5252 |
"Unexpected setcc expansion!"); |
--- |
5252 |
"Unexpected setcc expansion!"); |
--- |
| 5253 |
return NewLHS; |
0 |
5253 |
return NewLHS; |
0 |
| 5254 |
} |
--- |
5254 |
} |
--- |
| 5255 |
|
--- |
5255 |
|
--- |
| 5256 |
// Otherwise, update N to have the operands specified. |
--- |
5256 |
// Otherwise, update N to have the operands specified. |
--- |
| 5257 |
return SDValue( |
0 |
5257 |
return SDValue( |
0 |
| 5258 |
DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0); |
0 |
5258 |
DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0); |
0 |
| 5259 |
} |
--- |
5259 |
} |
--- |
| 5260 |
|
--- |
5260 |
|
--- |
| 5261 |
SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) { |
0 |
5261 |
SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) { |
0 |
| 5262 |
SDValue LHS = N->getOperand(0); |
0 |
5262 |
SDValue LHS = N->getOperand(0); |
0 |
| 5263 |
SDValue RHS = N->getOperand(1); |
0 |
5263 |
SDValue RHS = N->getOperand(1); |
0 |
| 5264 |
SDValue Carry = N->getOperand(2); |
0 |
5264 |
SDValue Carry = N->getOperand(2); |
0 |
| 5265 |
SDValue Cond = N->getOperand(3); |
0 |
5265 |
SDValue Cond = N->getOperand(3); |
0 |
| 5266 |
SDLoc dl = SDLoc(N); |
0 |
5266 |
SDLoc dl = SDLoc(N); |
0 |
| 5267 |
|
--- |
5267 |
|
--- |
| 5268 |
SDValue LHSLo, LHSHi, RHSLo, RHSHi; |
0 |
5268 |
SDValue LHSLo, LHSHi, RHSLo, RHSHi; |
0 |
| 5269 |
GetExpandedInteger(LHS, LHSLo, LHSHi); |
0 |
5269 |
GetExpandedInteger(LHS, LHSLo, LHSHi); |
0 |
| 5270 |
GetExpandedInteger(RHS, RHSLo, RHSHi); |
0 |
5270 |
GetExpandedInteger(RHS, RHSLo, RHSHi); |
0 |
| 5271 |
|
--- |
5271 |
|
--- |
| 5272 |
// Expand to a USUBO_CARRY for the low part and a SETCCCARRY for the high. |
--- |
5272 |
// Expand to a USUBO_CARRY for the low part and a SETCCCARRY for the high. |
--- |
| 5273 |
SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType()); |
0 |
5273 |
SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType()); |
0 |
| 5274 |
SDValue LowCmp = |
--- |
5274 |
SDValue LowCmp = |
--- |
| 5275 |
DAG.getNode(ISD::USUBO_CARRY, dl, VTList, LHSLo, RHSLo, Carry); |
0 |
5275 |
DAG.getNode(ISD::USUBO_CARRY, dl, VTList, LHSLo, RHSLo, Carry); |
0 |
| 5276 |
return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi, |
0 |
5276 |
return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi, |
0 |
| 5277 |
LowCmp.getValue(1), Cond); |
0 |
5277 |
LowCmp.getValue(1), Cond); |
0 |
| 5278 |
} |
0 |
5278 |
} |
0 |
| 5279 |
|
--- |
5279 |
|
--- |
| 5280 |
SDValue DAGTypeLegalizer::ExpandIntOp_SPLAT_VECTOR(SDNode *N) { |
0 |
5280 |
SDValue DAGTypeLegalizer::ExpandIntOp_SPLAT_VECTOR(SDNode *N) { |
0 |
| 5281 |
// Split the operand and replace with SPLAT_VECTOR_PARTS. |
--- |
5281 |
// Split the operand and replace with SPLAT_VECTOR_PARTS. |
--- |
| 5282 |
SDValue Lo, Hi; |
0 |
5282 |
SDValue Lo, Hi; |
0 |
| 5283 |
GetExpandedInteger(N->getOperand(0), Lo, Hi); |
0 |
5283 |
GetExpandedInteger(N->getOperand(0), Lo, Hi); |
0 |
| 5284 |
return DAG.getNode(ISD::SPLAT_VECTOR_PARTS, SDLoc(N), N->getValueType(0), Lo, |
0 |
5284 |
return DAG.getNode(ISD::SPLAT_VECTOR_PARTS, SDLoc(N), N->getValueType(0), Lo, |
0 |
| 5285 |
Hi); |
0 |
5285 |
Hi); |
0 |
| 5286 |
} |
--- |
5286 |
} |
--- |
| 5287 |
|
--- |
5287 |
|
--- |
| 5288 |
SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) { |
0 |
5288 |
SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) { |
0 |
| 5289 |
// The value being shifted is legal, but the shift amount is too big. |
--- |
5289 |
// The value being shifted is legal, but the shift amount is too big. |
--- |
| 5290 |
// It follows that either the result of the shift is undefined, or the |
--- |
5290 |
// It follows that either the result of the shift is undefined, or the |
--- |
| 5291 |
// upper half of the shift amount is zero. Just use the lower half. |
--- |
5291 |
// upper half of the shift amount is zero. Just use the lower half. |
--- |
| 5292 |
SDValue Lo, Hi; |
0 |
5292 |
SDValue Lo, Hi; |
0 |
| 5293 |
GetExpandedInteger(N->getOperand(1), Lo, Hi); |
0 |
5293 |
GetExpandedInteger(N->getOperand(1), Lo, Hi); |
0 |
| 5294 |
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0); |
0 |
5294 |
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0); |
0 |
| 5295 |
} |
--- |
5295 |
} |
--- |
| 5296 |
|
--- |
5296 |
|
--- |
| 5297 |
SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) { |
0 |
5297 |
SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) { |
0 |
| 5298 |
// The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This |
--- |
5298 |
// The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This |
--- |
| 5299 |
// surely makes pretty nice problems on 8/16 bit targets. Just truncate this |
--- |
5299 |
// surely makes pretty nice problems on 8/16 bit targets. Just truncate this |
--- |
| 5300 |
// constant to valid type. |
--- |
5300 |
// constant to valid type. |
--- |
| 5301 |
SDValue Lo, Hi; |
0 |
5301 |
SDValue Lo, Hi; |
0 |
| 5302 |
GetExpandedInteger(N->getOperand(0), Lo, Hi); |
0 |
5302 |
GetExpandedInteger(N->getOperand(0), Lo, Hi); |
0 |
| 5303 |
return SDValue(DAG.UpdateNodeOperands(N, Lo), 0); |
0 |
5303 |
return SDValue(DAG.UpdateNodeOperands(N, Lo), 0); |
0 |
| 5304 |
} |
--- |
5304 |
} |
--- |
| 5305 |
|
--- |
5305 |
|
--- |
| 5306 |
SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) { |
0 |
5306 |
SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) { |
0 |
| 5307 |
bool IsStrict = N->isStrictFPOpcode(); |
0 |
5307 |
bool IsStrict = N->isStrictFPOpcode(); |
0 |
| 5308 |
SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); |
0 |
5308 |
SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); |
0 |
| 5309 |
SDValue Op = N->getOperand(IsStrict ? 1 : 0); |
0 |
5309 |
SDValue Op = N->getOperand(IsStrict ? 1 : 0); |
0 |
| 5310 |
EVT DstVT = N->getValueType(0); |
0 |
5310 |
EVT DstVT = N->getValueType(0); |
0 |
| 5311 |
RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT); |
0 |
5311 |
RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT); |
0 |
| 5312 |
assert(LC != RTLIB::UNKNOWN_LIBCALL && |
0 |
5312 |
assert(LC != RTLIB::UNKNOWN_LIBCALL && |
0 |
| 5313 |
"Don't know how to expand this SINT_TO_FP!"); |
--- |
5313 |
"Don't know how to expand this SINT_TO_FP!"); |
--- |
| 5314 |
TargetLowering::MakeLibCallOptions CallOptions; |
0 |
5314 |
TargetLowering::MakeLibCallOptions CallOptions; |
0 |
| 5315 |
CallOptions.setSExt(true); |
0 |
5315 |
CallOptions.setSExt(true); |
0 |
| 5316 |
std::pair Tmp = |
--- |
5316 |
std::pair Tmp = |
--- |
| 5317 |
TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain); |
0 |
5317 |
TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain); |
0 |
| 5318 |
|
--- |
5318 |
|
--- |
| 5319 |
if (!IsStrict) |
0 |
5319 |
if (!IsStrict) |
0 |
| 5320 |
return Tmp.first; |
0 |
5320 |
return Tmp.first; |
0 |
| 5321 |
|
--- |
5321 |
|
--- |
| 5322 |
ReplaceValueWith(SDValue(N, 1), Tmp.second); |
0 |
5322 |
ReplaceValueWith(SDValue(N, 1), Tmp.second); |
0 |
| 5323 |
ReplaceValueWith(SDValue(N, 0), Tmp.first); |
0 |
5323 |
ReplaceValueWith(SDValue(N, 0), Tmp.first); |
0 |
| 5324 |
return SDValue(); |
0 |
5324 |
return SDValue(); |
0 |
| 5325 |
} |
--- |
5325 |
} |
--- |
| 5326 |
|
--- |
5326 |
|
--- |
| 5327 |
SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) { |
0 |
5327 |
SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) { |
0 |
| 5328 |
if (N->isAtomic()) { |
0 |
5328 |
if (N->isAtomic()) { |
0 |
| 5329 |
// It's typical to have larger CAS than atomic store instructions. |
--- |
5329 |
// It's typical to have larger CAS than atomic store instructions. |
--- |
| 5330 |
SDLoc dl(N); |
0 |
5330 |
SDLoc dl(N); |
0 |
| 5331 |
SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, |
0 |
5331 |
SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, |
0 |
| 5332 |
N->getMemoryVT(), |
--- |
5332 |
N->getMemoryVT(), |
--- |
| 5333 |
N->getOperand(0), N->getOperand(2), |
0 |
5333 |
N->getOperand(0), N->getOperand(2), |
0 |
| 5334 |
N->getOperand(1), |
0 |
5334 |
N->getOperand(1), |
0 |
| 5335 |
N->getMemOperand()); |
--- |
5335 |
N->getMemOperand()); |
--- |
| 5336 |
return Swap.getValue(1); |
0 |
5336 |
return Swap.getValue(1); |
0 |
| 5337 |
} |
0 |
5337 |
} |
0 |
| 5338 |
if (ISD::isNormalStore(N)) |
0 |
5338 |
if (ISD::isNormalStore(N)) |
0 |
| 5339 |
return ExpandOp_NormalStore(N, OpNo); |
0 |
5339 |
return ExpandOp_NormalStore(N, OpNo); |
0 |
| 5340 |
|
--- |
5340 |
|
--- |
| 5341 |
assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); |
0 |
5341 |
assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); |
0 |
| 5342 |
assert(OpNo == 1 && "Can only expand the stored value so far"); |
0 |
5342 |
assert(OpNo == 1 && "Can only expand the stored value so far"); |
0 |
| 5343 |
|
--- |
5343 |
|
--- |
| 5344 |
EVT VT = N->getOperand(1).getValueType(); |
0 |
5344 |
EVT VT = N->getOperand(1).getValueType(); |
0 |
| 5345 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); |
0 |
5345 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); |
0 |
| 5346 |
SDValue Ch = N->getChain(); |
0 |
5346 |
SDValue Ch = N->getChain(); |
0 |
| 5347 |
SDValue Ptr = N->getBasePtr(); |
0 |
5347 |
SDValue Ptr = N->getBasePtr(); |
0 |
| 5348 |
MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags(); |
0 |
5348 |
MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags(); |
0 |
| 5349 |
AAMDNodes AAInfo = N->getAAInfo(); |
0 |
5349 |
AAMDNodes AAInfo = N->getAAInfo(); |
0 |
| 5350 |
SDLoc dl(N); |
0 |
5350 |
SDLoc dl(N); |
0 |
| 5351 |
SDValue Lo, Hi; |
0 |
5351 |
SDValue Lo, Hi; |
0 |
| 5352 |
|
--- |
5352 |
|
--- |
| 5353 |
assert(NVT.isByteSized() && "Expanded type not byte sized!"); |
0 |
5353 |
assert(NVT.isByteSized() && "Expanded type not byte sized!"); |
0 |
| 5354 |
|
--- |
5354 |
|
--- |
| 5355 |
if (N->getMemoryVT().bitsLE(NVT)) { |
0 |
5355 |
if (N->getMemoryVT().bitsLE(NVT)) { |
0 |
| 5356 |
GetExpandedInteger(N->getValue(), Lo, Hi); |
0 |
5356 |
GetExpandedInteger(N->getValue(), Lo, Hi); |
0 |
| 5357 |
return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), |
0 |
5357 |
return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), |
0 |
| 5358 |
N->getMemoryVT(), N->getOriginalAlign(), MMOFlags, |
--- |
5358 |
N->getMemoryVT(), N->getOriginalAlign(), MMOFlags, |
--- |
| 5359 |
AAInfo); |
0 |
5359 |
AAInfo); |
0 |
| 5360 |
} |
--- |
5360 |
} |
--- |
| 5361 |
|
--- |
5361 |
|
--- |
| 5362 |
if (DAG.getDataLayout().isLittleEndian()) { |
0 |
5362 |
if (DAG.getDataLayout().isLittleEndian()) { |
0 |
| 5363 |
// Little-endian - low bits are at low addresses. |
--- |
5363 |
// Little-endian - low bits are at low addresses. |
--- |
| 5364 |
GetExpandedInteger(N->getValue(), Lo, Hi); |
0 |
5364 |
GetExpandedInteger(N->getValue(), Lo, Hi); |
0 |
| 5365 |
|
--- |
5365 |
|
--- |
| 5366 |
Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), |
0 |
5366 |
Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), |
0 |
| 5367 |
N->getOriginalAlign(), MMOFlags, AAInfo); |
--- |
5367 |
N->getOriginalAlign(), MMOFlags, AAInfo); |
--- |
| 5368 |
|
--- |
5368 |
|
--- |
| 5369 |
unsigned ExcessBits = |
--- |
5369 |
unsigned ExcessBits = |
--- |
| 5370 |
N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); |
0 |
5370 |
N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); |
0 |
| 5371 |
EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); |
0 |
5371 |
EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); |
0 |
| 5372 |
|
--- |
5372 |
|
--- |
| 5373 |
// Increment the pointer to the other half. |
--- |
5373 |
// Increment the pointer to the other half. |
--- |
| 5374 |
unsigned IncrementSize = NVT.getSizeInBits()/8; |
0 |
5374 |
unsigned IncrementSize = NVT.getSizeInBits()/8; |
0 |
| 5375 |
Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize)); |
0 |
5375 |
Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize)); |
0 |
| 5376 |
Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, |
0 |
5376 |
Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, |
0 |
| 5377 |
N->getPointerInfo().getWithOffset(IncrementSize), |
0 |
5377 |
N->getPointerInfo().getWithOffset(IncrementSize), |
0 |
| 5378 |
NEVT, N->getOriginalAlign(), MMOFlags, AAInfo); |
--- |
5378 |
NEVT, N->getOriginalAlign(), MMOFlags, AAInfo); |
--- |
| 5379 |
return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); |
0 |
5379 |
return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); |
0 |
| 5380 |
} |
--- |
5380 |
} |
--- |
| 5381 |
|
--- |
5381 |
|
--- |
| 5382 |
// Big-endian - high bits are at low addresses. Favor aligned stores at |
--- |
5382 |
// Big-endian - high bits are at low addresses. Favor aligned stores at |
--- |
| 5383 |
// the cost of some bit-fiddling. |
--- |
5383 |
// the cost of some bit-fiddling. |
--- |
| 5384 |
GetExpandedInteger(N->getValue(), Lo, Hi); |
0 |
5384 |
GetExpandedInteger(N->getValue(), Lo, Hi); |
0 |
| 5385 |
|
--- |
5385 |
|
--- |
| 5386 |
EVT ExtVT = N->getMemoryVT(); |
0 |
5386 |
EVT ExtVT = N->getMemoryVT(); |
0 |
| 5387 |
unsigned EBytes = ExtVT.getStoreSize(); |
0 |
5387 |
unsigned EBytes = ExtVT.getStoreSize(); |
0 |
| 5388 |
unsigned IncrementSize = NVT.getSizeInBits()/8; |
0 |
5388 |
unsigned IncrementSize = NVT.getSizeInBits()/8; |
0 |
| 5389 |
unsigned ExcessBits = (EBytes - IncrementSize)*8; |
0 |
5389 |
unsigned ExcessBits = (EBytes - IncrementSize)*8; |
0 |
| 5390 |
EVT HiVT = EVT::getIntegerVT(*DAG.getContext(), |
0 |
5390 |
EVT HiVT = EVT::getIntegerVT(*DAG.getContext(), |
0 |
| 5391 |
ExtVT.getSizeInBits() - ExcessBits); |
0 |
5391 |
ExtVT.getSizeInBits() - ExcessBits); |
0 |
| 5392 |
|
--- |
5392 |
|
--- |
| 5393 |
if (ExcessBits < NVT.getSizeInBits()) { |
0 |
5393 |
if (ExcessBits < NVT.getSizeInBits()) { |
0 |
| 5394 |
// Transfer high bits from the top of Lo to the bottom of Hi. |
--- |
5394 |
// Transfer high bits from the top of Lo to the bottom of Hi. |
--- |
| 5395 |
Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi, |
0 |
5395 |
Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi, |
0 |
| 5396 |
DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl, |
0 |
5396 |
DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl, |
0 |
| 5397 |
TLI.getPointerTy(DAG.getDataLayout()))); |
0 |
5397 |
TLI.getPointerTy(DAG.getDataLayout()))); |
0 |
| 5398 |
Hi = DAG.getNode( |
0 |
5398 |
Hi = DAG.getNode( |
0 |
| 5399 |
ISD::OR, dl, NVT, Hi, |
--- |
5399 |
ISD::OR, dl, NVT, Hi, |
--- |
| 5400 |
DAG.getNode(ISD::SRL, dl, NVT, Lo, |
0 |
5400 |
DAG.getNode(ISD::SRL, dl, NVT, Lo, |
0 |
| 5401 |
DAG.getConstant(ExcessBits, dl, |
0 |
5401 |
DAG.getConstant(ExcessBits, dl, |
0 |
| 5402 |
TLI.getPointerTy(DAG.getDataLayout())))); |
0 |
5402 |
TLI.getPointerTy(DAG.getDataLayout())))); |
0 |
| 5403 |
} |
--- |
5403 |
} |
--- |
| 5404 |
|
--- |
5404 |
|
--- |
| 5405 |
// Store both the high bits and maybe some of the low bits. |
--- |
5405 |
// Store both the high bits and maybe some of the low bits. |
--- |
| 5406 |
Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT, |
0 |
5406 |
Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT, |
0 |
| 5407 |
N->getOriginalAlign(), MMOFlags, AAInfo); |
--- |
5407 |
N->getOriginalAlign(), MMOFlags, AAInfo); |
--- |
| 5408 |
|
--- |
5408 |
|
--- |
| 5409 |
// Increment the pointer to the other half. |
--- |
5409 |
// Increment the pointer to the other half. |
--- |
| 5410 |
Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize)); |
0 |
5410 |
Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize)); |
0 |
| 5411 |
// Store the lowest ExcessBits bits in the second half. |
--- |
5411 |
// Store the lowest ExcessBits bits in the second half. |
--- |
| 5412 |
Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr, |
0 |
5412 |
Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr, |
0 |
| 5413 |
N->getPointerInfo().getWithOffset(IncrementSize), |
0 |
5413 |
N->getPointerInfo().getWithOffset(IncrementSize), |
0 |
| 5414 |
EVT::getIntegerVT(*DAG.getContext(), ExcessBits), |
0 |
5414 |
EVT::getIntegerVT(*DAG.getContext(), ExcessBits), |
0 |
| 5415 |
N->getOriginalAlign(), MMOFlags, AAInfo); |
--- |
5415 |
N->getOriginalAlign(), MMOFlags, AAInfo); |
--- |
| 5416 |
return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); |
0 |
5416 |
return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); |
0 |
| 5417 |
} |
0 |
5417 |
} |
0 |
| 5418 |
|
--- |
5418 |
|
--- |
| 5419 |
SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) { |
0 |
5419 |
SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) { |
0 |
| 5420 |
SDValue InL, InH; |
0 |
5420 |
SDValue InL, InH; |
0 |
| 5421 |
GetExpandedInteger(N->getOperand(0), InL, InH); |
0 |
5421 |
GetExpandedInteger(N->getOperand(0), InL, InH); |
0 |
| 5422 |
// Just truncate the low part of the source. |
--- |
5422 |
// Just truncate the low part of the source. |
--- |
| 5423 |
return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL); |
0 |
5423 |
return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL); |
0 |
| 5424 |
} |
--- |
5424 |
} |
--- |
| 5425 |
|
--- |
5425 |
|
--- |
| 5426 |
SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) { |
0 |
5426 |
SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) { |
0 |
| 5427 |
bool IsStrict = N->isStrictFPOpcode(); |
0 |
5427 |
bool IsStrict = N->isStrictFPOpcode(); |
0 |
| 5428 |
SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); |
0 |
5428 |
SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); |
0 |
| 5429 |
SDValue Op = N->getOperand(IsStrict ? 1 : 0); |
0 |
5429 |
SDValue Op = N->getOperand(IsStrict ? 1 : 0); |
0 |
| 5430 |
EVT DstVT = N->getValueType(0); |
0 |
5430 |
EVT DstVT = N->getValueType(0); |
0 |
| 5431 |
RTLIB::Libcall LC = RTLIB::getUINTTOFP(Op.getValueType(), DstVT); |
0 |
5431 |
RTLIB::Libcall LC = RTLIB::getUINTTOFP(Op.getValueType(), DstVT); |
0 |
| 5432 |
assert(LC != RTLIB::UNKNOWN_LIBCALL && |
0 |
5432 |
assert(LC != RTLIB::UNKNOWN_LIBCALL && |
0 |
| 5433 |
"Don't know how to expand this UINT_TO_FP!"); |
--- |
5433 |
"Don't know how to expand this UINT_TO_FP!"); |
--- |
| 5434 |
TargetLowering::MakeLibCallOptions CallOptions; |
0 |
5434 |
TargetLowering::MakeLibCallOptions CallOptions; |
0 |
| 5435 |
CallOptions.setSExt(true); |
0 |
5435 |
CallOptions.setSExt(true); |
0 |
| 5436 |
std::pair Tmp = |
--- |
5436 |
std::pair Tmp = |
--- |
| 5437 |
TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain); |
0 |
5437 |
TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain); |
0 |
| 5438 |
|
--- |
5438 |
|
--- |
| 5439 |
if (!IsStrict) |
0 |
5439 |
if (!IsStrict) |
0 |
| 5440 |
return Tmp.first; |
0 |
5440 |
return Tmp.first; |
0 |
| 5441 |
|
--- |
5441 |
|
--- |
| 5442 |
ReplaceValueWith(SDValue(N, 1), Tmp.second); |
0 |
5442 |
ReplaceValueWith(SDValue(N, 1), Tmp.second); |
0 |
| 5443 |
ReplaceValueWith(SDValue(N, 0), Tmp.first); |
0 |
5443 |
ReplaceValueWith(SDValue(N, 0), Tmp.first); |
0 |
| 5444 |
return SDValue(); |
0 |
5444 |
return SDValue(); |
0 |
| 5445 |
} |
--- |
5445 |
} |
--- |
| 5446 |
|
--- |
5446 |
|
--- |
| 5447 |
SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) { |
0 |
5447 |
SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) { |
0 |
| 5448 |
SDLoc dl(N); |
0 |
5448 |
SDLoc dl(N); |
0 |
| 5449 |
SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, |
0 |
5449 |
SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, |
0 |
| 5450 |
cast(N)->getMemoryVT(), |
0 |
5450 |
cast(N)->getMemoryVT(), |
0 |
| 5451 |
N->getOperand(0), |
0 |
5451 |
N->getOperand(0), |
0 |
| 5452 |
N->getOperand(1), N->getOperand(2), |
0 |
5452 |
N->getOperand(1), N->getOperand(2), |
0 |
| 5453 |
cast(N)->getMemOperand()); |
0 |
5453 |
cast(N)->getMemOperand()); |
0 |
| 5454 |
return Swap.getValue(1); |
0 |
5454 |
return Swap.getValue(1); |
0 |
| 5455 |
} |
0 |
5455 |
} |
0 |
| 5456 |
|
--- |
5456 |
|
--- |
| 5457 |
SDValue DAGTypeLegalizer::ExpandIntOp_VP_STRIDED(SDNode *N, unsigned OpNo) { |
0 |
5457 |
SDValue DAGTypeLegalizer::ExpandIntOp_VP_STRIDED(SDNode *N, unsigned OpNo) { |
0 |
| 5458 |
assert((N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) || |
0 |
5458 |
assert((N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) || |
0 |
| 5459 |
(N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4)); |
--- |
5459 |
(N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4)); |
--- |
| 5460 |
|
--- |
5460 |
|
--- |
| 5461 |
SDValue Hi; // The upper half is dropped out. |
0 |
5461 |
SDValue Hi; // The upper half is dropped out. |
0 |
| 5462 |
SmallVector NewOps(N->op_begin(), N->op_end()); |
0 |
5462 |
SmallVector NewOps(N->op_begin(), N->op_end()); |
0 |
| 5463 |
GetExpandedInteger(NewOps[OpNo], NewOps[OpNo], Hi); |
0 |
5463 |
GetExpandedInteger(NewOps[OpNo], NewOps[OpNo], Hi); |
0 |
| 5464 |
|
--- |
5464 |
|
--- |
| 5465 |
return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); |
0 |
5465 |
return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); |
0 |
| 5466 |
} |
0 |
5466 |
} |
0 |
| 5467 |
|
--- |
5467 |
|
--- |
| 5468 |
SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SPLICE(SDNode *N) { |
0 |
5468 |
SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SPLICE(SDNode *N) { |
0 |
| 5469 |
SDLoc dl(N); |
0 |
5469 |
SDLoc dl(N); |
0 |
| 5470 |
|
--- |
5470 |
|
--- |
| 5471 |
SDValue V0 = GetPromotedInteger(N->getOperand(0)); |
0 |
5471 |
SDValue V0 = GetPromotedInteger(N->getOperand(0)); |
0 |
| 5472 |
SDValue V1 = GetPromotedInteger(N->getOperand(1)); |
0 |
5472 |
SDValue V1 = GetPromotedInteger(N->getOperand(1)); |
0 |
| 5473 |
EVT OutVT = V0.getValueType(); |
0 |
5473 |
EVT OutVT = V0.getValueType(); |
0 |
| 5474 |
|
--- |
5474 |
|
--- |
| 5475 |
return DAG.getNode(ISD::VECTOR_SPLICE, dl, OutVT, V0, V1, N->getOperand(2)); |
0 |
5475 |
return DAG.getNode(ISD::VECTOR_SPLICE, dl, OutVT, V0, V1, N->getOperand(2)); |
0 |
| 5476 |
} |
0 |
5476 |
} |
0 |
| 5477 |
|
--- |
5477 |
|
--- |
| 5478 |
SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(SDNode *N) { |
0 |
5478 |
SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(SDNode *N) { |
0 |
| 5479 |
SDLoc dl(N); |
0 |
5479 |
SDLoc dl(N); |
0 |
| 5480 |
|
--- |
5480 |
|
--- |
| 5481 |
SDValue V0 = GetPromotedInteger(N->getOperand(0)); |
0 |
5481 |
SDValue V0 = GetPromotedInteger(N->getOperand(0)); |
0 |
| 5482 |
SDValue V1 = GetPromotedInteger(N->getOperand(1)); |
0 |
5482 |
SDValue V1 = GetPromotedInteger(N->getOperand(1)); |
0 |
| 5483 |
EVT ResVT = V0.getValueType(); |
0 |
5483 |
EVT ResVT = V0.getValueType(); |
0 |
| 5484 |
SDValue Res = DAG.getNode(N->getOpcode(), dl, |
0 |
5484 |
SDValue Res = DAG.getNode(N->getOpcode(), dl, |
0 |
| 5485 |
DAG.getVTList(ResVT, ResVT), V0, V1); |
0 |
5485 |
DAG.getVTList(ResVT, ResVT), V0, V1); |
0 |
| 5486 |
SetPromotedInteger(SDValue(N, 0), Res.getValue(0)); |
0 |
5486 |
SetPromotedInteger(SDValue(N, 0), Res.getValue(0)); |
0 |
| 5487 |
SetPromotedInteger(SDValue(N, 1), Res.getValue(1)); |
0 |
5487 |
SetPromotedInteger(SDValue(N, 1), Res.getValue(1)); |
0 |
| 5488 |
return SDValue(); |
0 |
5488 |
return SDValue(); |
0 |
| 5489 |
} |
0 |
5489 |
} |
0 |
| 5490 |
|
--- |
5490 |
|
--- |
| 5491 |
SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) { |
0 |
5491 |
SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) { |
0 |
| 5492 |
|
--- |
5492 |
|
--- |
| 5493 |
EVT OutVT = N->getValueType(0); |
0 |
5493 |
EVT OutVT = N->getValueType(0); |
0 |
| 5494 |
EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); |
0 |
5494 |
EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); |
0 |
| 5495 |
assert(NOutVT.isVector() && "This type must be promoted to a vector type"); |
0 |
5495 |
assert(NOutVT.isVector() && "This type must be promoted to a vector type"); |
0 |
| 5496 |
EVT NOutVTElem = NOutVT.getVectorElementType(); |
0 |
5496 |
EVT NOutVTElem = NOutVT.getVectorElementType(); |
0 |
| 5497 |
|
--- |
5497 |
|
--- |
| 5498 |
SDLoc dl(N); |
0 |
5498 |
SDLoc dl(N); |
0 |
| 5499 |
SDValue BaseIdx = N->getOperand(1); |
0 |
5499 |
SDValue BaseIdx = N->getOperand(1); |
0 |
| 5500 |
|
--- |
5500 |
|
--- |
| 5501 |
// TODO: We may be able to use this for types other than scalable |
--- |
5501 |
// TODO: We may be able to use this for types other than scalable |
--- |
| 5502 |
// vectors and fix those tests that expect BUILD_VECTOR to be used |
--- |
5502 |
// vectors and fix those tests that expect BUILD_VECTOR to be used |
--- |
| 5503 |
if (OutVT.isScalableVector()) { |
0 |
5503 |
if (OutVT.isScalableVector()) { |
0 |
| 5504 |
SDValue InOp0 = N->getOperand(0); |
0 |
5504 |
SDValue InOp0 = N->getOperand(0); |
0 |
| 5505 |
EVT InVT = InOp0.getValueType(); |
0 |
5505 |
EVT InVT = InOp0.getValueType(); |
0 |
| 5506 |
|
--- |
5506 |
|
--- |
| 5507 |
// Try and extract from a smaller type so that it eventually falls |
--- |
5507 |
// Try and extract from a smaller type so that it eventually falls |
--- |
| 5508 |
// into the promotion code below. |
--- |
5508 |
// into the promotion code below. |
--- |
| 5509 |
if (getTypeAction(InVT) == TargetLowering::TypeSplitVector || |
0 |
5509 |
if (getTypeAction(InVT) == TargetLowering::TypeSplitVector || |
0 |
| 5510 |
getTypeAction(InVT) == TargetLowering::TypeLegal) { |
0 |
5510 |
getTypeAction(InVT) == TargetLowering::TypeLegal) { |
0 |
| 5511 |
EVT NInVT = InVT.getHalfNumVectorElementsVT(*DAG.getContext()); |
0 |
5511 |
EVT NInVT = InVT.getHalfNumVectorElementsVT(*DAG.getContext()); |
0 |
| 5512 |
unsigned NElts = NInVT.getVectorMinNumElements(); |
0 |
5512 |
unsigned NElts = NInVT.getVectorMinNumElements(); |
0 |
| 5513 |
uint64_t IdxVal = cast(BaseIdx)->getZExtValue(); |
0 |
5513 |
uint64_t IdxVal = cast(BaseIdx)->getZExtValue(); |
0 |
| 5514 |
|
--- |
5514 |
|
--- |
| 5515 |
SDValue Step1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NInVT, InOp0, |
0 |
5515 |
SDValue Step1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NInVT, InOp0, |
0 |
| 5516 |
DAG.getConstant(alignDown(IdxVal, NElts), dl, |
0 |
5516 |
DAG.getConstant(alignDown(IdxVal, NElts), dl, |
0 |
| 5517 |
BaseIdx.getValueType())); |
--- |
5517 |
BaseIdx.getValueType())); |
--- |
| 5518 |
SDValue Step2 = DAG.getNode( |
0 |
5518 |
SDValue Step2 = DAG.getNode( |
0 |
| 5519 |
ISD::EXTRACT_SUBVECTOR, dl, OutVT, Step1, |
--- |
5519 |
ISD::EXTRACT_SUBVECTOR, dl, OutVT, Step1, |
--- |
| 5520 |
DAG.getConstant(IdxVal % NElts, dl, BaseIdx.getValueType())); |
0 |
5520 |
DAG.getConstant(IdxVal % NElts, dl, BaseIdx.getValueType())); |
0 |
| 5521 |
return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Step2); |
0 |
5521 |
return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Step2); |
0 |
| 5522 |
} |
--- |
5522 |
} |
--- |
| 5523 |
|
--- |
5523 |
|
--- |
| 5524 |
// Try and extract from a widened type. |
--- |
5524 |
// Try and extract from a widened type. |
--- |
| 5525 |
if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { |
0 |
5525 |
if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { |
0 |
| 5526 |
SDValue Ops[] = {GetWidenedVector(InOp0), BaseIdx}; |
0 |
5526 |
SDValue Ops[] = {GetWidenedVector(InOp0), BaseIdx}; |
0 |
| 5527 |
SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), OutVT, Ops); |
0 |
5527 |
SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), OutVT, Ops); |
0 |
| 5528 |
return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Ext); |
0 |
5528 |
return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Ext); |
0 |
| 5529 |
} |
--- |
5529 |
} |
--- |
| 5530 |
|
--- |
5530 |
|
--- |
| 5531 |
// Promote operands and see if this is handled by target lowering, |
--- |
5531 |
// Promote operands and see if this is handled by target lowering, |
--- |
| 5532 |
// Otherwise, use the BUILD_VECTOR approach below |
--- |
5532 |
// Otherwise, use the BUILD_VECTOR approach below |
--- |
| 5533 |
if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) { |
0 |
5533 |
if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) { |
0 |
| 5534 |
// Collect the (promoted) operands |
--- |
5534 |
// Collect the (promoted) operands |
--- |
| 5535 |
SDValue Ops[] = { GetPromotedInteger(InOp0), BaseIdx }; |
0 |
5535 |
SDValue Ops[] = { GetPromotedInteger(InOp0), BaseIdx }; |
0 |
| 5536 |
|
--- |
5536 |
|
--- |
| 5537 |
EVT PromEltVT = Ops[0].getValueType().getVectorElementType(); |
0 |
5537 |
EVT PromEltVT = Ops[0].getValueType().getVectorElementType(); |
0 |
| 5538 |
assert(PromEltVT.bitsLE(NOutVTElem) && |
0 |
5538 |
assert(PromEltVT.bitsLE(NOutVTElem) && |
0 |
| 5539 |
"Promoted operand has an element type greater than result"); |
--- |
5539 |
"Promoted operand has an element type greater than result"); |
--- |
| 5540 |
|
--- |
5540 |
|
--- |
| 5541 |
EVT ExtVT = NOutVT.changeVectorElementType(PromEltVT); |
0 |
5541 |
EVT ExtVT = NOutVT.changeVectorElementType(PromEltVT); |
0 |
| 5542 |
SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), ExtVT, Ops); |
0 |
5542 |
SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), ExtVT, Ops); |
0 |
| 5543 |
return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Ext); |
0 |
5543 |
return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Ext); |
0 |
| 5544 |
} |
--- |
5544 |
} |
--- |
| 5545 |
} |
--- |
5545 |
} |
--- |
| 5546 |
|
--- |
5546 |
|
--- |
| 5547 |
if (OutVT.isScalableVector()) |
0 |
5547 |
if (OutVT.isScalableVector()) |
0 |
| 5548 |
report_fatal_error("Unable to promote scalable types using BUILD_VECTOR"); |
0 |
5548 |
report_fatal_error("Unable to promote scalable types using BUILD_VECTOR"); |
0 |
| 5549 |
|
--- |
5549 |
|
--- |
| 5550 |
SDValue InOp0 = N->getOperand(0); |
0 |
5550 |
SDValue InOp0 = N->getOperand(0); |
0 |
| 5551 |
if (getTypeAction(InOp0.getValueType()) == TargetLowering::TypePromoteInteger) |
0 |
5551 |
if (getTypeAction(InOp0.getValueType()) == TargetLowering::TypePromoteInteger) |
0 |
| 5552 |
InOp0 = GetPromotedInteger(N->getOperand(0)); |
0 |
5552 |
InOp0 = GetPromotedInteger(N->getOperand(0)); |
0 |
| 5553 |
|
--- |
5553 |
|
--- |
| 5554 |
EVT InVT = InOp0.getValueType(); |
0 |
5554 |
EVT InVT = InOp0.getValueType(); |
0 |
| 5555 |
|
--- |
5555 |
|
--- |
| 5556 |
unsigned OutNumElems = OutVT.getVectorNumElements(); |
0 |
5556 |
unsigned OutNumElems = OutVT.getVectorNumElements(); |
0 |
| 5557 |
SmallVector Ops; |
0 |
5557 |
SmallVector Ops; |
0 |
| 5558 |
Ops.reserve(OutNumElems); |
0 |
5558 |
Ops.reserve(OutNumElems); |
0 |
| 5559 |
for (unsigned i = 0; i != OutNumElems; ++i) { |
0 |
5559 |
for (unsigned i = 0; i != OutNumElems; ++i) { |
0 |
| 5560 |
|
--- |
5560 |
|
--- |
| 5561 |
// Extract the element from the original vector. |
--- |
5561 |
// Extract the element from the original vector. |
--- |
| 5562 |
SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(), |
0 |
5562 |
SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(), |
0 |
| 5563 |
BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType())); |
0 |
5563 |
BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType())); |
0 |
| 5564 |
SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, |
0 |
5564 |
SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, |
0 |
| 5565 |
InVT.getVectorElementType(), N->getOperand(0), Index); |
0 |
5565 |
InVT.getVectorElementType(), N->getOperand(0), Index); |
0 |
| 5566 |
|
--- |
5566 |
|
--- |
| 5567 |
SDValue Op = DAG.getAnyExtOrTrunc(Ext, dl, NOutVTElem); |
0 |
5567 |
SDValue Op = DAG.getAnyExtOrTrunc(Ext, dl, NOutVTElem); |
0 |
| 5568 |
// Insert the converted element to the new vector. |
--- |
5568 |
// Insert the converted element to the new vector. |
--- |
| 5569 |
Ops.push_back(Op); |
0 |
5569 |
Ops.push_back(Op); |
0 |
| 5570 |
} |
--- |
5570 |
} |
--- |
| 5571 |
|
--- |
5571 |
|
--- |
| 5572 |
return DAG.getBuildVector(NOutVT, dl, Ops); |
0 |
5572 |
return DAG.getBuildVector(NOutVT, dl, Ops); |
0 |
| 5573 |
} |
0 |
5573 |
} |
0 |
| 5574 |
|
--- |
5574 |
|
--- |
| 5575 |
SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_SUBVECTOR(SDNode *N) { |
0 |
5575 |
SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_SUBVECTOR(SDNode *N) { |
0 |
| 5576 |
EVT OutVT = N->getValueType(0); |
0 |
5576 |
EVT OutVT = N->getValueType(0); |
0 |
| 5577 |
EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); |
0 |
5577 |
EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); |
0 |
| 5578 |
assert(NOutVT.isVector() && "This type must be promoted to a vector type"); |
0 |
5578 |
assert(NOutVT.isVector() && "This type must be promoted to a vector type"); |
0 |
| 5579 |
|
--- |
5579 |
|
--- |
| 5580 |
SDLoc dl(N); |
0 |
5580 |
SDLoc dl(N); |
0 |
| 5581 |
SDValue Vec = N->getOperand(0); |
0 |
5581 |
SDValue Vec = N->getOperand(0); |
0 |
| 5582 |
SDValue SubVec = N->getOperand(1); |
0 |
5582 |
SDValue SubVec = N->getOperand(1); |
0 |
| 5583 |
SDValue Idx = N->getOperand(2); |
0 |
5583 |
SDValue Idx = N->getOperand(2); |
0 |
| 5584 |
|
--- |
5584 |
|
--- |
| 5585 |
EVT SubVecVT = SubVec.getValueType(); |
0 |
5585 |
EVT SubVecVT = SubVec.getValueType(); |
0 |
| 5586 |
EVT NSubVT = |
--- |
5586 |
EVT NSubVT = |
--- |
| 5587 |
EVT::getVectorVT(*DAG.getContext(), NOutVT.getVectorElementType(), |
0 |
5587 |
EVT::getVectorVT(*DAG.getContext(), NOutVT.getVectorElementType(), |
0 |
| 5588 |
SubVecVT.getVectorElementCount()); |
--- |
5588 |
SubVecVT.getVectorElementCount()); |
--- |
| 5589 |
|
--- |
5589 |
|
--- |
| 5590 |
Vec = GetPromotedInteger(Vec); |
0 |
5590 |
Vec = GetPromotedInteger(Vec); |
0 |
| 5591 |
SubVec = DAG.getNode(ISD::ANY_EXTEND, dl, NSubVT, SubVec); |
0 |
5591 |
SubVec = DAG.getNode(ISD::ANY_EXTEND, dl, NSubVT, SubVec); |
0 |
| 5592 |
|
--- |
5592 |
|
--- |
| 5593 |
return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, NOutVT, Vec, SubVec, Idx); |
0 |
5593 |
return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, NOutVT, Vec, SubVec, Idx); |
0 |
| 5594 |
} |
0 |
5594 |
} |
0 |
| 5595 |
|
--- |
5595 |
|
--- |
| 5596 |
SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_REVERSE(SDNode *N) { |
0 |
5596 |
SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_REVERSE(SDNode *N) { |
0 |
| 5597 |
SDLoc dl(N); |
0 |
5597 |
SDLoc dl(N); |
0 |
| 5598 |
|
--- |
5598 |
|
--- |
| 5599 |
SDValue V0 = GetPromotedInteger(N->getOperand(0)); |
0 |
5599 |
SDValue V0 = GetPromotedInteger(N->getOperand(0)); |
0 |
| 5600 |
EVT OutVT = V0.getValueType(); |
0 |
5600 |
EVT OutVT = V0.getValueType(); |
0 |
| 5601 |
|
--- |
5601 |
|
--- |
| 5602 |
return DAG.getNode(ISD::VECTOR_REVERSE, dl, OutVT, V0); |
0 |
5602 |
return DAG.getNode(ISD::VECTOR_REVERSE, dl, OutVT, V0); |
0 |
| 5603 |
} |
0 |
5603 |
} |
0 |
| 5604 |
|
--- |
5604 |
|
--- |
| 5605 |
SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) { |
0 |
5605 |
SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) { |
0 |
| 5606 |
ShuffleVectorSDNode *SV = cast(N); |
0 |
5606 |
ShuffleVectorSDNode *SV = cast(N); |
0 |
| 5607 |
EVT VT = N->getValueType(0); |
0 |
5607 |
EVT VT = N->getValueType(0); |
0 |
| 5608 |
SDLoc dl(N); |
0 |
5608 |
SDLoc dl(N); |
0 |
| 5609 |
|
--- |
5609 |
|
--- |
| 5610 |
ArrayRef NewMask = SV->getMask().slice(0, VT.getVectorNumElements()); |
0 |
5610 |
ArrayRef NewMask = SV->getMask().slice(0, VT.getVectorNumElements()); |
0 |
| 5611 |
|
--- |
5611 |
|
--- |
| 5612 |
SDValue V0 = GetPromotedInteger(N->getOperand(0)); |
0 |
5612 |
SDValue V0 = GetPromotedInteger(N->getOperand(0)); |
0 |
| 5613 |
SDValue V1 = GetPromotedInteger(N->getOperand(1)); |
0 |
5613 |
SDValue V1 = GetPromotedInteger(N->getOperand(1)); |
0 |
| 5614 |
EVT OutVT = V0.getValueType(); |
0 |
5614 |
EVT OutVT = V0.getValueType(); |
0 |
| 5615 |
|
--- |
5615 |
|
--- |
| 5616 |
return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask); |
0 |
5616 |
return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask); |
0 |
| 5617 |
} |
0 |
5617 |
} |
0 |
| 5618 |
|
--- |
5618 |
|
--- |
| 5619 |
SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) { |
0 |
5619 |
SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) { |
0 |
| 5620 |
EVT OutVT = N->getValueType(0); |
0 |
5620 |
EVT OutVT = N->getValueType(0); |
0 |
| 5621 |
EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); |
0 |
5621 |
EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); |
0 |
| 5622 |
assert(NOutVT.isVector() && "This type must be promoted to a vector type"); |
0 |
5622 |
assert(NOutVT.isVector() && "This type must be promoted to a vector type"); |
0 |
| 5623 |
unsigned NumElems = N->getNumOperands(); |
0 |
5623 |
unsigned NumElems = N->getNumOperands(); |
0 |
| 5624 |
EVT NOutVTElem = NOutVT.getVectorElementType(); |
0 |
5624 |
EVT NOutVTElem = NOutVT.getVectorElementType(); |
0 |
| 5625 |
TargetLoweringBase::BooleanContent NOutBoolType = TLI.getBooleanContents(NOutVT); |
0 |
5625 |
TargetLoweringBase::BooleanContent NOutBoolType = TLI.getBooleanContents(NOutVT); |
0 |
| 5626 |
unsigned NOutExtOpc = TargetLowering::getExtendForContent(NOutBoolType); |
0 |
5626 |
unsigned NOutExtOpc = TargetLowering::getExtendForContent(NOutBoolType); |
0 |
| 5627 |
SDLoc dl(N); |
0 |
5627 |
SDLoc dl(N); |
0 |
| 5628 |
|
--- |
5628 |
|
--- |
| 5629 |
SmallVector Ops; |
0 |
5629 |
SmallVector Ops; |
0 |
| 5630 |
Ops.reserve(NumElems); |
0 |
5630 |
Ops.reserve(NumElems); |
0 |
| 5631 |
for (unsigned i = 0; i != NumElems; ++i) { |
0 |
5631 |
for (unsigned i = 0; i != NumElems; ++i) { |
0 |
| 5632 |
SDValue Op = N->getOperand(i); |
0 |
5632 |
SDValue Op = N->getOperand(i); |
0 |
| 5633 |
EVT OpVT = Op.getValueType(); |
0 |
5633 |
EVT OpVT = Op.getValueType(); |
0 |
| 5634 |
// BUILD_VECTOR integer operand types are allowed to be larger than the |
--- |
5634 |
// BUILD_VECTOR integer operand types are allowed to be larger than the |
--- |
| 5635 |
// result's element type. This may still be true after the promotion. For |
--- |
5635 |
// result's element type. This may still be true after the promotion. For |
--- |
| 5636 |
// example, we might be promoting ( = BV , , ...) to |
--- |
5636 |
// example, we might be promoting ( = BV , , ...) to |
--- |
| 5637 |
// (v?i16 = BV , , ...), and we can't any_extend to . |
--- |
5637 |
// (v?i16 = BV , , ...), and we can't any_extend to . |
--- |
| 5638 |
if (OpVT.bitsLT(NOutVTElem)) { |
0 |
5638 |
if (OpVT.bitsLT(NOutVTElem)) { |
0 |
| 5639 |
unsigned ExtOpc = ISD::ANY_EXTEND; |
0 |
5639 |
unsigned ExtOpc = ISD::ANY_EXTEND; |
0 |
| 5640 |
// Attempt to extend constant bool vectors to match target's BooleanContent. |
--- |
5640 |
// Attempt to extend constant bool vectors to match target's BooleanContent. |
--- |
| 5641 |
// While not necessary, this improves chances of the constant correctly |
--- |
5641 |
// While not necessary, this improves chances of the constant correctly |
--- |
| 5642 |
// folding with compare results (e.g. for NOT patterns). |
--- |
5642 |
// folding with compare results (e.g. for NOT patterns). |
--- |
| 5643 |
if (OpVT == MVT::i1 && Op.getOpcode() == ISD::Constant) |
0 |
5643 |
if (OpVT == MVT::i1 && Op.getOpcode() == ISD::Constant) |
0 |
| 5644 |
ExtOpc = NOutExtOpc; |
0 |
5644 |
ExtOpc = NOutExtOpc; |
0 |
| 5645 |
Op = DAG.getNode(ExtOpc, dl, NOutVTElem, Op); |
0 |
5645 |
Op = DAG.getNode(ExtOpc, dl, NOutVTElem, Op); |
0 |
| 5646 |
} |
--- |
5646 |
} |
--- |
| 5647 |
Ops.push_back(Op); |
0 |
5647 |
Ops.push_back(Op); |
0 |
| 5648 |
} |
--- |
5648 |
} |
--- |
| 5649 |
|
--- |
5649 |
|
--- |
| 5650 |
return DAG.getBuildVector(NOutVT, dl, Ops); |
0 |
5650 |
return DAG.getBuildVector(NOutVT, dl, Ops); |
0 |
| 5651 |
} |
0 |
5651 |
} |
0 |
| 5652 |
|
--- |
5652 |
|
--- |
| 5653 |
SDValue DAGTypeLegalizer::PromoteIntRes_ScalarOp(SDNode *N) { |
0 |
5653 |
SDValue DAGTypeLegalizer::PromoteIntRes_ScalarOp(SDNode *N) { |
0 |
| 5654 |
|
--- |
5654 |
|
--- |
| 5655 |
SDLoc dl(N); |
0 |
5655 |
SDLoc dl(N); |
0 |
| 5656 |
|
--- |
5656 |
|
--- |
| 5657 |
assert(!N->getOperand(0).getValueType().isVector() && |
0 |
5657 |
assert(!N->getOperand(0).getValueType().isVector() && |
0 |
| 5658 |
"Input must be a scalar"); |
--- |
5658 |
"Input must be a scalar"); |
--- |
| 5659 |
|
--- |
5659 |
|
--- |
| 5660 |
EVT OutVT = N->getValueType(0); |
0 |
5660 |
EVT OutVT = N->getValueType(0); |
0 |
| 5661 |
EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); |
0 |
5661 |
EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); |
0 |
| 5662 |
assert(NOutVT.isVector() && "This type must be promoted to a vector type"); |
0 |
5662 |
assert(NOutVT.isVector() && "This type must be promoted to a vector type"); |
0 |
| 5663 |
EVT NOutElemVT = NOutVT.getVectorElementType(); |
0 |
5663 |
EVT NOutElemVT = NOutVT.getVectorElementType(); |
0 |
| 5664 |
|
--- |
5664 |
|
--- |
| 5665 |
SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutElemVT, N->getOperand(0)); |
0 |
5665 |
SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutElemVT, N->getOperand(0)); |
0 |
| 5666 |
|
--- |
5666 |
|
--- |
| 5667 |
return DAG.getNode(N->getOpcode(), dl, NOutVT, Op); |
0 |
5667 |
return DAG.getNode(N->getOpcode(), dl, NOutVT, Op); |
0 |
| 5668 |
} |
0 |
5668 |
} |
0 |
| 5669 |
|
--- |
5669 |
|
--- |
| 5670 |
SDValue DAGTypeLegalizer::PromoteIntRes_STEP_VECTOR(SDNode *N) { |
0 |
5670 |
SDValue DAGTypeLegalizer::PromoteIntRes_STEP_VECTOR(SDNode *N) { |
0 |
| 5671 |
SDLoc dl(N); |
0 |
5671 |
SDLoc dl(N); |
0 |
| 5672 |
EVT OutVT = N->getValueType(0); |
0 |
5672 |
EVT OutVT = N->getValueType(0); |
0 |
| 5673 |
EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); |
0 |
5673 |
EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); |
0 |
| 5674 |
assert(NOutVT.isScalableVector() && |
0 |
5674 |
assert(NOutVT.isScalableVector() && |
0 |
| 5675 |
"Type must be promoted to a scalable vector type"); |
--- |
5675 |
"Type must be promoted to a scalable vector type"); |
--- |
| 5676 |
const APInt &StepVal = N->getConstantOperandAPInt(0); |
0 |
5676 |
const APInt &StepVal = N->getConstantOperandAPInt(0); |
0 |
| 5677 |
return DAG.getStepVector(dl, NOutVT, |
0 |
5677 |
return DAG.getStepVector(dl, NOutVT, |
0 |
| 5678 |
StepVal.sext(NOutVT.getScalarSizeInBits())); |
0 |
5678 |
StepVal.sext(NOutVT.getScalarSizeInBits())); |
0 |
| 5679 |
} |
0 |
5679 |
} |
0 |
| 5680 |
|
--- |
5680 |
|
--- |
| 5681 |
SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) { |
0 |
5681 |
SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) { |
0 |
| 5682 |
SDLoc dl(N); |
0 |
5682 |
SDLoc dl(N); |
0 |
| 5683 |
|
--- |
5683 |
|
--- |
| 5684 |
EVT OutVT = N->getValueType(0); |
0 |
5684 |
EVT OutVT = N->getValueType(0); |
0 |
| 5685 |
EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); |
0 |
5685 |
EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); |
0 |
| 5686 |
assert(NOutVT.isVector() && "This type must be promoted to a vector type"); |
0 |
5686 |
assert(NOutVT.isVector() && "This type must be promoted to a vector type"); |
0 |
| 5687 |
|
--- |
5687 |
|
--- |
| 5688 |
unsigned NumOperands = N->getNumOperands(); |
0 |
5688 |
unsigned NumOperands = N->getNumOperands(); |
0 |
| 5689 |
unsigned NumOutElem = NOutVT.getVectorMinNumElements(); |
0 |
5689 |
unsigned NumOutElem = NOutVT.getVectorMinNumElements(); |
0 |
| 5690 |
EVT OutElemTy = NOutVT.getVectorElementType(); |
0 |
5690 |
EVT OutElemTy = NOutVT.getVectorElementType(); |
0 |
| 5691 |
if (OutVT.isScalableVector()) { |
0 |
5691 |
if (OutVT.isScalableVector()) { |
0 |
| 5692 |
// Find the largest promoted element type for each of the operands. |
--- |
5692 |
// Find the largest promoted element type for each of the operands. |
--- |
| 5693 |
SDUse *MaxSizedValue = std::max_element( |
0 |
5693 |
SDUse *MaxSizedValue = std::max_element( |
0 |
| 5694 |
N->op_begin(), N->op_end(), [](const SDValue &A, const SDValue &B) { |
0 |
5694 |
N->op_begin(), N->op_end(), [](const SDValue &A, const SDValue &B) { |
0 |
| 5695 |
EVT AVT = A.getValueType().getVectorElementType(); |
0 |
5695 |
EVT AVT = A.getValueType().getVectorElementType(); |
0 |
| 5696 |
EVT BVT = B.getValueType().getVectorElementType(); |
0 |
5696 |
EVT BVT = B.getValueType().getVectorElementType(); |
0 |
| 5697 |
return AVT.getScalarSizeInBits() < BVT.getScalarSizeInBits(); |
0 |
5697 |
return AVT.getScalarSizeInBits() < BVT.getScalarSizeInBits(); |
0 |
| 5698 |
}); |
--- |
5698 |
}); |
--- |
| 5699 |
EVT MaxElementVT = MaxSizedValue->getValueType().getVectorElementType(); |
0 |
5699 |
EVT MaxElementVT = MaxSizedValue->getValueType().getVectorElementType(); |
0 |
| 5700 |
|
--- |
5700 |
|
--- |
| 5701 |
// Then promote all vectors to the largest element type. |
--- |
5701 |
// Then promote all vectors to the largest element type. |
--- |
| 5702 |
SmallVector Ops; |
0 |
5702 |
SmallVector Ops; |
0 |
| 5703 |
for (unsigned I = 0; I < NumOperands; ++I) { |
0 |
5703 |
for (unsigned I = 0; I < NumOperands; ++I) { |
0 |
| 5704 |
SDValue Op = N->getOperand(I); |
0 |
5704 |
SDValue Op = N->getOperand(I); |
0 |
| 5705 |
EVT OpVT = Op.getValueType(); |
0 |
5705 |
EVT OpVT = Op.getValueType(); |
0 |
| 5706 |
if (getTypeAction(OpVT) == TargetLowering::TypePromoteInteger) |
0 |
5706 |
if (getTypeAction(OpVT) == TargetLowering::TypePromoteInteger) |
0 |
| 5707 |
Op = GetPromotedInteger(Op); |
0 |
5707 |
Op = GetPromotedInteger(Op); |
0 |
| 5708 |
else |
--- |
5708 |
else |
--- |
| 5709 |
assert(getTypeAction(OpVT) == TargetLowering::TypeLegal && |
0 |
5709 |
assert(getTypeAction(OpVT) == TargetLowering::TypeLegal && |
0 |
| 5710 |
"Unhandled legalization type"); |
--- |
5710 |
"Unhandled legalization type"); |
--- |
| 5711 |
|
--- |
5711 |
|
--- |
| 5712 |
if (OpVT.getVectorElementType().getScalarSizeInBits() < |
0 |
5712 |
if (OpVT.getVectorElementType().getScalarSizeInBits() < |
0 |
| 5713 |
MaxElementVT.getScalarSizeInBits()) |
0 |
5713 |
MaxElementVT.getScalarSizeInBits()) |
0 |
| 5714 |
Op = DAG.getAnyExtOrTrunc(Op, dl, |
0 |
5714 |
Op = DAG.getAnyExtOrTrunc(Op, dl, |
0 |
| 5715 |
OpVT.changeVectorElementType(MaxElementVT)); |
--- |
5715 |
OpVT.changeVectorElementType(MaxElementVT)); |
--- |
| 5716 |
Ops.push_back(Op); |
0 |
5716 |
Ops.push_back(Op); |
0 |
| 5717 |
} |
--- |
5717 |
} |
--- |
| 5718 |
|
--- |
5718 |
|
--- |
| 5719 |
// Do the CONCAT on the promoted type and finally truncate to (the promoted) |
--- |
5719 |
// Do the CONCAT on the promoted type and finally truncate to (the promoted) |
--- |
| 5720 |
// NOutVT. |
--- |
5720 |
// NOutVT. |
--- |
| 5721 |
return DAG.getAnyExtOrTrunc( |
0 |
5721 |
return DAG.getAnyExtOrTrunc( |
0 |
| 5722 |
DAG.getNode(ISD::CONCAT_VECTORS, dl, |
0 |
5722 |
DAG.getNode(ISD::CONCAT_VECTORS, dl, |
0 |
| 5723 |
OutVT.changeVectorElementType(MaxElementVT), Ops), |
--- |
5723 |
OutVT.changeVectorElementType(MaxElementVT), Ops), |
--- |
| 5724 |
dl, NOutVT); |
0 |
5724 |
dl, NOutVT); |
0 |
| 5725 |
} |
0 |
5725 |
} |
0 |
| 5726 |
|
--- |
5726 |
|
--- |
| 5727 |
unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements(); |
0 |
5727 |
unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements(); |
0 |
| 5728 |
assert(NumElem * NumOperands == NumOutElem && |
0 |
5728 |
assert(NumElem * NumOperands == NumOutElem && |
0 |
| 5729 |
"Unexpected number of elements"); |
--- |
5729 |
"Unexpected number of elements"); |
--- |
| 5730 |
|
--- |
5730 |
|
--- |
| 5731 |
// Take the elements from the first vector. |
--- |
5731 |
// Take the elements from the first vector. |
--- |
| 5732 |
SmallVector Ops(NumOutElem); |
0 |
5732 |
SmallVector Ops(NumOutElem); |
0 |
| 5733 |
for (unsigned i = 0; i < NumOperands; ++i) { |
0 |
5733 |
for (unsigned i = 0; i < NumOperands; ++i) { |
0 |
| 5734 |
SDValue Op = N->getOperand(i); |
0 |
5734 |
SDValue Op = N->getOperand(i); |
0 |
| 5735 |
if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger) |
0 |
5735 |
if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger) |
0 |
| 5736 |
Op = GetPromotedInteger(Op); |
0 |
5736 |
Op = GetPromotedInteger(Op); |
0 |
| 5737 |
EVT SclrTy = Op.getValueType().getVectorElementType(); |
0 |
5737 |
EVT SclrTy = Op.getValueType().getVectorElementType(); |
0 |
| 5738 |
assert(NumElem == Op.getValueType().getVectorNumElements() && |
0 |
5738 |
assert(NumElem == Op.getValueType().getVectorNumElements() && |
0 |
| 5739 |
"Unexpected number of elements"); |
--- |
5739 |
"Unexpected number of elements"); |
--- |
| 5740 |
|
--- |
5740 |
|
--- |
| 5741 |
for (unsigned j = 0; j < NumElem; ++j) { |
0 |
5741 |
for (unsigned j = 0; j < NumElem; ++j) { |
0 |
| 5742 |
SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Op, |
0 |
5742 |
SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Op, |
0 |
| 5743 |
DAG.getVectorIdxConstant(j, dl)); |
0 |
5743 |
DAG.getVectorIdxConstant(j, dl)); |
0 |
| 5744 |
Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy); |
0 |
5744 |
Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy); |
0 |
| 5745 |
} |
--- |
5745 |
} |
--- |
| 5746 |
} |
--- |
5746 |
} |
--- |
| 5747 |
|
--- |
5747 |
|
--- |
| 5748 |
return DAG.getBuildVector(NOutVT, dl, Ops); |
0 |
5748 |
return DAG.getBuildVector(NOutVT, dl, Ops); |
0 |
| 5749 |
} |
0 |
5749 |
} |
0 |
| 5750 |
|
--- |
5750 |
|
--- |
| 5751 |
SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) { |
0 |
5751 |
SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) { |
0 |
| 5752 |
EVT VT = N->getValueType(0); |
0 |
5752 |
EVT VT = N->getValueType(0); |
0 |
| 5753 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); |
0 |
5753 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); |
0 |
| 5754 |
assert(NVT.isVector() && "This type must be promoted to a vector type"); |
0 |
5754 |
assert(NVT.isVector() && "This type must be promoted to a vector type"); |
0 |
| 5755 |
|
--- |
5755 |
|
--- |
| 5756 |
SDLoc dl(N); |
0 |
5756 |
SDLoc dl(N); |
0 |
| 5757 |
|
--- |
5757 |
|
--- |
| 5758 |
// For operands whose TypeAction is to promote, extend the promoted node |
--- |
5758 |
// For operands whose TypeAction is to promote, extend the promoted node |
--- |
| 5759 |
// appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion |
--- |
5759 |
// appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion |
--- |
| 5760 |
// type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to |
--- |
5760 |
// type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to |
--- |
| 5761 |
// type.. |
--- |
5761 |
// type.. |
--- |
| 5762 |
if (getTypeAction(N->getOperand(0).getValueType()) |
0 |
5762 |
if (getTypeAction(N->getOperand(0).getValueType()) |
0 |
| 5763 |
== TargetLowering::TypePromoteInteger) { |
0 |
5763 |
== TargetLowering::TypePromoteInteger) { |
0 |
| 5764 |
SDValue Promoted; |
0 |
5764 |
SDValue Promoted; |
0 |
| 5765 |
|
--- |
5765 |
|
--- |
| 5766 |
switch(N->getOpcode()) { |
0 |
5766 |
switch(N->getOpcode()) { |
0 |
| 5767 |
case ISD::SIGN_EXTEND_VECTOR_INREG: |
0 |
5767 |
case ISD::SIGN_EXTEND_VECTOR_INREG: |
0 |
| 5768 |
Promoted = SExtPromotedInteger(N->getOperand(0)); |
0 |
5768 |
Promoted = SExtPromotedInteger(N->getOperand(0)); |
0 |
| 5769 |
break; |
0 |
5769 |
break; |
0 |
| 5770 |
case ISD::ZERO_EXTEND_VECTOR_INREG: |
0 |
5770 |
case ISD::ZERO_EXTEND_VECTOR_INREG: |
0 |
| 5771 |
Promoted = ZExtPromotedInteger(N->getOperand(0)); |
0 |
5771 |
Promoted = ZExtPromotedInteger(N->getOperand(0)); |
0 |
| 5772 |
break; |
0 |
5772 |
break; |
0 |
| 5773 |
case ISD::ANY_EXTEND_VECTOR_INREG: |
0 |
5773 |
case ISD::ANY_EXTEND_VECTOR_INREG: |
0 |
| 5774 |
Promoted = GetPromotedInteger(N->getOperand(0)); |
0 |
5774 |
Promoted = GetPromotedInteger(N->getOperand(0)); |
0 |
| 5775 |
break; |
0 |
5775 |
break; |
0 |
| 5776 |
default: |
0 |
5776 |
default: |
0 |
| 5777 |
llvm_unreachable("Node has unexpected Opcode"); |
0 |
5777 |
llvm_unreachable("Node has unexpected Opcode"); |
0 |
| 5778 |
} |
--- |
5778 |
} |
--- |
| 5779 |
return DAG.getNode(N->getOpcode(), dl, NVT, Promoted); |
0 |
5779 |
return DAG.getNode(N->getOpcode(), dl, NVT, Promoted); |
0 |
| 5780 |
} |
--- |
5780 |
} |
--- |
| 5781 |
|
--- |
5781 |
|
--- |
| 5782 |
// Directly extend to the appropriate transform-to type. |
--- |
5782 |
// Directly extend to the appropriate transform-to type. |
--- |
| 5783 |
return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); |
0 |
5783 |
return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); |
0 |
| 5784 |
} |
0 |
5784 |
} |
0 |
| 5785 |
|
--- |
5785 |
|
--- |
| 5786 |
SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) { |
0 |
5786 |
SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) { |
0 |
| 5787 |
EVT OutVT = N->getValueType(0); |
0 |
5787 |
EVT OutVT = N->getValueType(0); |
0 |
| 5788 |
EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); |
0 |
5788 |
EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); |
0 |
| 5789 |
assert(NOutVT.isVector() && "This type must be promoted to a vector type"); |
0 |
5789 |
assert(NOutVT.isVector() && "This type must be promoted to a vector type"); |
0 |
| 5790 |
|
--- |
5790 |
|
--- |
| 5791 |
EVT NOutVTElem = NOutVT.getVectorElementType(); |
0 |
5791 |
EVT NOutVTElem = NOutVT.getVectorElementType(); |
0 |
| 5792 |
|
--- |
5792 |
|
--- |
| 5793 |
SDLoc dl(N); |
0 |
5793 |
SDLoc dl(N); |
0 |
| 5794 |
SDValue V0 = GetPromotedInteger(N->getOperand(0)); |
0 |
5794 |
SDValue V0 = GetPromotedInteger(N->getOperand(0)); |
0 |
| 5795 |
|
--- |
5795 |
|
--- |
| 5796 |
SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl, |
0 |
5796 |
SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl, |
0 |
| 5797 |
NOutVTElem, N->getOperand(1)); |
0 |
5797 |
NOutVTElem, N->getOperand(1)); |
0 |
| 5798 |
return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT, |
0 |
5798 |
return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT, |
0 |
| 5799 |
V0, ConvElem, N->getOperand(2)); |
0 |
5799 |
V0, ConvElem, N->getOperand(2)); |
0 |
| 5800 |
} |
0 |
5800 |
} |
0 |
| 5801 |
|
--- |
5801 |
|
--- |
| 5802 |
SDValue DAGTypeLegalizer::PromoteIntRes_VECREDUCE(SDNode *N) { |
0 |
5802 |
SDValue DAGTypeLegalizer::PromoteIntRes_VECREDUCE(SDNode *N) { |
0 |
| 5803 |
// The VECREDUCE result size may be larger than the element size, so |
--- |
5803 |
// The VECREDUCE result size may be larger than the element size, so |
--- |
| 5804 |
// we can simply change the result type. |
--- |
5804 |
// we can simply change the result type. |
--- |
| 5805 |
SDLoc dl(N); |
0 |
5805 |
SDLoc dl(N); |
0 |
| 5806 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
5806 |
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
0 |
| 5807 |
return DAG.getNode(N->getOpcode(), dl, NVT, N->ops()); |
0 |
5807 |
return DAG.getNode(N->getOpcode(), dl, NVT, N->ops()); |
0 |
| 5808 |
} |
0 |
5808 |
} |
0 |
| 5809 |
|
--- |
5809 |
|
--- |
| 5810 |
SDValue DAGTypeLegalizer::PromoteIntRes_VP_REDUCE(SDNode *N) { |
0 |
5810 |
SDValue DAGTypeLegalizer::PromoteIntRes_VP_REDUCE(SDNode *N) { |
0 |
| 5811 |
// The VP_REDUCE result size may be larger than the element size, so we can |
--- |
5811 |
// The VP_REDUCE result size may be larger than the element size, so we can |
--- |
| 5812 |
// simply change the result type. However the start value and result must be |
--- |
5812 |
// simply change the result type. However the start value and result must be |
--- |
| 5813 |
// the same. |
--- |
5813 |
// the same. |
--- |
| 5814 |
SDLoc DL(N); |
0 |
5814 |
SDLoc DL(N); |
0 |
| 5815 |
SDValue Start = PromoteIntOpVectorReduction(N, N->getOperand(0)); |
0 |
5815 |
SDValue Start = PromoteIntOpVectorReduction(N, N->getOperand(0)); |
0 |
| 5816 |
return DAG.getNode(N->getOpcode(), DL, Start.getValueType(), Start, |
0 |
5816 |
return DAG.getNode(N->getOpcode(), DL, Start.getValueType(), Start, |
0 |
| 5817 |
N->getOperand(1), N->getOperand(2), N->getOperand(3)); |
0 |
5817 |
N->getOperand(1), N->getOperand(2), N->getOperand(3)); |
0 |
| 5818 |
} |
0 |
5818 |
} |
0 |
| 5819 |
|
--- |
5819 |
|
--- |
| 5820 |
SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) { |
0 |
5820 |
SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) { |
0 |
| 5821 |
SDLoc dl(N); |
0 |
5821 |
SDLoc dl(N); |
0 |
| 5822 |
SDValue V0 = GetPromotedInteger(N->getOperand(0)); |
0 |
5822 |
SDValue V0 = GetPromotedInteger(N->getOperand(0)); |
0 |
| 5823 |
SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl, |
0 |
5823 |
SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl, |
0 |
| 5824 |
TLI.getVectorIdxTy(DAG.getDataLayout())); |
0 |
5824 |
TLI.getVectorIdxTy(DAG.getDataLayout())); |
0 |
| 5825 |
SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, |
0 |
5825 |
SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, |
0 |
| 5826 |
V0->getValueType(0).getScalarType(), V0, V1); |
0 |
5826 |
V0->getValueType(0).getScalarType(), V0, V1); |
0 |
| 5827 |
|
--- |
5827 |
|
--- |
| 5828 |
// EXTRACT_VECTOR_ELT can return types which are wider than the incoming |
--- |
5828 |
// EXTRACT_VECTOR_ELT can return types which are wider than the incoming |
--- |
| 5829 |
// element types. If this is the case then we need to expand the outgoing |
--- |
5829 |
// element types. If this is the case then we need to expand the outgoing |
--- |
| 5830 |
// value and not truncate it. |
--- |
5830 |
// value and not truncate it. |
--- |
| 5831 |
return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0)); |
0 |
5831 |
return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0)); |
0 |
| 5832 |
} |
0 |
5832 |
} |
0 |
| 5833 |
|
--- |
5833 |
|
--- |
| 5834 |
SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_SUBVECTOR(SDNode *N) { |
0 |
5834 |
SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_SUBVECTOR(SDNode *N) { |
0 |
| 5835 |
SDLoc dl(N); |
0 |
5835 |
SDLoc dl(N); |
0 |
| 5836 |
// The result type is equal to the first input operand's type, so the |
--- |
5836 |
// The result type is equal to the first input operand's type, so the |
--- |
| 5837 |
// type that needs promoting must be the second source vector. |
--- |
5837 |
// type that needs promoting must be the second source vector. |
--- |
| 5838 |
SDValue V0 = N->getOperand(0); |
0 |
5838 |
SDValue V0 = N->getOperand(0); |
0 |
| 5839 |
SDValue V1 = GetPromotedInteger(N->getOperand(1)); |
0 |
5839 |
SDValue V1 = GetPromotedInteger(N->getOperand(1)); |
0 |
| 5840 |
SDValue Idx = N->getOperand(2); |
0 |
5840 |
SDValue Idx = N->getOperand(2); |
0 |
| 5841 |
EVT PromVT = EVT::getVectorVT(*DAG.getContext(), |
0 |
5841 |
EVT PromVT = EVT::getVectorVT(*DAG.getContext(), |
0 |
| 5842 |
V1.getValueType().getVectorElementType(), |
0 |
5842 |
V1.getValueType().getVectorElementType(), |
0 |
| 5843 |
V0.getValueType().getVectorElementCount()); |
0 |
5843 |
V0.getValueType().getVectorElementCount()); |
0 |
| 5844 |
V0 = DAG.getAnyExtOrTrunc(V0, dl, PromVT); |
0 |
5844 |
V0 = DAG.getAnyExtOrTrunc(V0, dl, PromVT); |
0 |
| 5845 |
SDValue Ext = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, PromVT, V0, V1, Idx); |
0 |
5845 |
SDValue Ext = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, PromVT, V0, V1, Idx); |
0 |
| 5846 |
return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0)); |
0 |
5846 |
return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0)); |
0 |
| 5847 |
} |
0 |
5847 |
} |
0 |
| 5848 |
|
--- |
5848 |
|
--- |
| 5849 |
SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) { |
0 |
5849 |
SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) { |
0 |
| 5850 |
SDLoc dl(N); |
0 |
5850 |
SDLoc dl(N); |
0 |
| 5851 |
SDValue V0 = GetPromotedInteger(N->getOperand(0)); |
0 |
5851 |
SDValue V0 = GetPromotedInteger(N->getOperand(0)); |
0 |
| 5852 |
MVT InVT = V0.getValueType().getSimpleVT(); |
0 |
5852 |
MVT InVT = V0.getValueType().getSimpleVT(); |
0 |
| 5853 |
MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(), |
0 |
5853 |
MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(), |
0 |
| 5854 |
N->getValueType(0).getVectorNumElements()); |
0 |
5854 |
N->getValueType(0).getVectorNumElements()); |
0 |
| 5855 |
SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1)); |
0 |
5855 |
SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1)); |
0 |
| 5856 |
return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext); |
0 |
5856 |
return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext); |
0 |
| 5857 |
} |
0 |
5857 |
} |
0 |
| 5858 |
|
--- |
5858 |
|
--- |
| 5859 |
SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) { |
0 |
5859 |
SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) { |
0 |
| 5860 |
SDLoc dl(N); |
0 |
5860 |
SDLoc dl(N); |
0 |
| 5861 |
|
--- |
5861 |
|
--- |
| 5862 |
EVT ResVT = N->getValueType(0); |
0 |
5862 |
EVT ResVT = N->getValueType(0); |
0 |
| 5863 |
unsigned NumElems = N->getNumOperands(); |
0 |
5863 |
unsigned NumElems = N->getNumOperands(); |
0 |
| 5864 |
|
--- |
5864 |
|
--- |
| 5865 |
if (ResVT.isScalableVector()) { |
0 |
5865 |
if (ResVT.isScalableVector()) { |
0 |
| 5866 |
SDValue ResVec = DAG.getUNDEF(ResVT); |
0 |
5866 |
SDValue ResVec = DAG.getUNDEF(ResVT); |
0 |
| 5867 |
|
--- |
5867 |
|
--- |
| 5868 |
for (unsigned OpIdx = 0; OpIdx < NumElems; ++OpIdx) { |
0 |
5868 |
for (unsigned OpIdx = 0; OpIdx < NumElems; ++OpIdx) { |
0 |
| 5869 |
SDValue Op = N->getOperand(OpIdx); |
0 |
5869 |
SDValue Op = N->getOperand(OpIdx); |
0 |
| 5870 |
unsigned OpNumElts = Op.getValueType().getVectorMinNumElements(); |
0 |
5870 |
unsigned OpNumElts = Op.getValueType().getVectorMinNumElements(); |
0 |
| 5871 |
ResVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, ResVec, Op, |
0 |
5871 |
ResVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, ResVec, Op, |
0 |
| 5872 |
DAG.getIntPtrConstant(OpIdx * OpNumElts, dl)); |
0 |
5872 |
DAG.getIntPtrConstant(OpIdx * OpNumElts, dl)); |
0 |
| 5873 |
} |
--- |
5873 |
} |
--- |
| 5874 |
|
--- |
5874 |
|
--- |
| 5875 |
return ResVec; |
0 |
5875 |
return ResVec; |
0 |
| 5876 |
} |
--- |
5876 |
} |
--- |
| 5877 |
|
--- |
5877 |
|
--- |
| 5878 |
EVT RetSclrTy = N->getValueType(0).getVectorElementType(); |
0 |
5878 |
EVT RetSclrTy = N->getValueType(0).getVectorElementType(); |
0 |
| 5879 |
|
--- |
5879 |
|
--- |
| 5880 |
SmallVector NewOps; |
0 |
5880 |
SmallVector NewOps; |
0 |
| 5881 |
NewOps.reserve(NumElems); |
0 |
5881 |
NewOps.reserve(NumElems); |
0 |
| 5882 |
|
--- |
5882 |
|
--- |
| 5883 |
// For each incoming vector |
--- |
5883 |
// For each incoming vector |
--- |
| 5884 |
for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) { |
0 |
5884 |
for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) { |
0 |
| 5885 |
SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx)); |
0 |
5885 |
SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx)); |
0 |
| 5886 |
EVT SclrTy = Incoming->getValueType(0).getVectorElementType(); |
0 |
5886 |
EVT SclrTy = Incoming->getValueType(0).getVectorElementType(); |
0 |
| 5887 |
unsigned NumElem = Incoming->getValueType(0).getVectorNumElements(); |
0 |
5887 |
unsigned NumElem = Incoming->getValueType(0).getVectorNumElements(); |
0 |
| 5888 |
|
--- |
5888 |
|
--- |
| 5889 |
for (unsigned i=0; i
| 0 |
5889 |
for (unsigned i=0; i
| 0 |
| |
| 5890 |
// Extract element from incoming vector |
--- |
5890 |
// Extract element from incoming vector |
--- |
| 5891 |
SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming, |
0 |
5891 |
SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming, |
0 |
| 5892 |
DAG.getVectorIdxConstant(i, dl)); |
0 |
5892 |
DAG.getVectorIdxConstant(i, dl)); |
0 |
| 5893 |
SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex); |
0 |
5893 |
SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex); |
0 |
| 5894 |
NewOps.push_back(Tr); |
0 |
5894 |
NewOps.push_back(Tr); |
0 |
| 5895 |
} |
--- |
5895 |
} |
--- |
| 5896 |
} |
--- |
5896 |
} |
--- |
| 5897 |
|
--- |
5897 |
|
--- |
| 5898 |
return DAG.getBuildVector(N->getValueType(0), dl, NewOps); |
0 |
5898 |
return DAG.getBuildVector(N->getValueType(0), dl, NewOps); |
0 |
| 5899 |
} |
0 |
5899 |
} |
0 |
| 5900 |
|
--- |
5900 |
|
--- |
| 5901 |
SDValue DAGTypeLegalizer::ExpandIntOp_STACKMAP(SDNode *N, unsigned OpNo) { |
0 |
5901 |
SDValue DAGTypeLegalizer::ExpandIntOp_STACKMAP(SDNode *N, unsigned OpNo) { |
0 |
| 5902 |
assert(OpNo > 1); |
0 |
5902 |
assert(OpNo > 1); |
0 |
| 5903 |
SDValue Op = N->getOperand(OpNo); |
0 |
5903 |
SDValue Op = N->getOperand(OpNo); |
0 |
| 5904 |
|
--- |
5904 |
|
--- |
| 5905 |
// FIXME: Non-constant operands are not yet handled: |
--- |
5905 |
// FIXME: Non-constant operands are not yet handled: |
--- |
| 5906 |
// - https://github.com/llvm/llvm-project/issues/26431 |
--- |
5906 |
// - https://github.com/llvm/llvm-project/issues/26431 |
--- |
| 5907 |
// - https://github.com/llvm/llvm-project/issues/55957 |
--- |
5907 |
// - https://github.com/llvm/llvm-project/issues/55957 |
--- |
| 5908 |
ConstantSDNode *CN = dyn_cast(Op); |
0 |
5908 |
ConstantSDNode *CN = dyn_cast(Op); |
0 |
| 5909 |
if (!CN) |
0 |
5909 |
if (!CN) |
0 |
| 5910 |
return SDValue(); |
0 |
5910 |
return SDValue(); |
0 |
| 5911 |
|
--- |
5911 |
|
--- |
| 5912 |
// Copy operands before the one being expanded. |
--- |
5912 |
// Copy operands before the one being expanded. |
--- |
| 5913 |
SmallVector NewOps; |
0 |
5913 |
SmallVector NewOps; |
0 |
| 5914 |
for (unsigned I = 0; I < OpNo; I++) |
0 |
5914 |
for (unsigned I = 0; I < OpNo; I++) |
0 |
| 5915 |
NewOps.push_back(N->getOperand(I)); |
0 |
5915 |
NewOps.push_back(N->getOperand(I)); |
0 |
| 5916 |
|
--- |
5916 |
|
--- |
| 5917 |
EVT Ty = Op.getValueType(); |
0 |
5917 |
EVT Ty = Op.getValueType(); |
0 |
| 5918 |
SDLoc DL = SDLoc(N); |
0 |
5918 |
SDLoc DL = SDLoc(N); |
0 |
| 5919 |
if (CN->getConstantIntValue()->getValue().getActiveBits() < 64) { |
0 |
5919 |
if (CN->getConstantIntValue()->getValue().getActiveBits() < 64) { |
0 |
| 5920 |
NewOps.push_back( |
0 |
5920 |
NewOps.push_back( |
0 |
| 5921 |
DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64)); |
0 |
5921 |
DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64)); |
0 |
| 5922 |
NewOps.push_back(DAG.getTargetConstant(CN->getZExtValue(), DL, Ty)); |
0 |
5922 |
NewOps.push_back(DAG.getTargetConstant(CN->getZExtValue(), DL, Ty)); |
0 |
| 5923 |
} else { |
--- |
5923 |
} else { |
--- |
| 5924 |
// FIXME: https://github.com/llvm/llvm-project/issues/55609 |
--- |
5924 |
// FIXME: https://github.com/llvm/llvm-project/issues/55609 |
--- |
| 5925 |
return SDValue(); |
0 |
5925 |
return SDValue(); |
0 |
| 5926 |
} |
--- |
5926 |
} |
--- |
| 5927 |
|
--- |
5927 |
|
--- |
| 5928 |
// Copy remaining operands. |
--- |
5928 |
// Copy remaining operands. |
--- |
| 5929 |
for (unsigned I = OpNo + 1; I < N->getNumOperands(); I++) |
0 |
5929 |
for (unsigned I = OpNo + 1; I < N->getNumOperands(); I++) |
0 |
| 5930 |
NewOps.push_back(N->getOperand(I)); |
0 |
5930 |
NewOps.push_back(N->getOperand(I)); |
0 |
| 5931 |
|
--- |
5931 |
|
--- |
| 5932 |
SDValue NewNode = DAG.getNode(N->getOpcode(), DL, N->getVTList(), NewOps); |
0 |
5932 |
SDValue NewNode = DAG.getNode(N->getOpcode(), DL, N->getVTList(), NewOps); |
0 |
| 5933 |
|
--- |
5933 |
|
--- |
| 5934 |
for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++) |
0 |
5934 |
for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++) |
0 |
| 5935 |
ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum)); |
0 |
5935 |
ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum)); |
0 |
| 5936 |
|
--- |
5936 |
|
--- |
| 5937 |
return SDValue(); // Signal that we have replaced the node already. |
0 |
5937 |
return SDValue(); // Signal that we have replaced the node already. |
0 |
| 5938 |
} |
0 |
5938 |
} |
0 |
| 5939 |
|
--- |
5939 |
|
--- |
| 5940 |
SDValue DAGTypeLegalizer::ExpandIntOp_PATCHPOINT(SDNode *N, unsigned OpNo) { |
0 |
5940 |
SDValue DAGTypeLegalizer::ExpandIntOp_PATCHPOINT(SDNode *N, unsigned OpNo) { |
0 |
| 5941 |
assert(OpNo >= 7); |
0 |
5941 |
assert(OpNo >= 7); |
0 |
| 5942 |
SDValue Op = N->getOperand(OpNo); |
0 |
5942 |
SDValue Op = N->getOperand(OpNo); |
0 |
| 5943 |
|
--- |
5943 |
|
--- |
| 5944 |
// FIXME: Non-constant operands are not yet handled: |
--- |
5944 |
// FIXME: Non-constant operands are not yet handled: |
--- |
| 5945 |
// - https://github.com/llvm/llvm-project/issues/26431 |
--- |
5945 |
// - https://github.com/llvm/llvm-project/issues/26431 |
--- |
| 5946 |
// - https://github.com/llvm/llvm-project/issues/55957 |
--- |
5946 |
// - https://github.com/llvm/llvm-project/issues/55957 |
--- |
| 5947 |
ConstantSDNode *CN = dyn_cast(Op); |
0 |
5947 |
ConstantSDNode *CN = dyn_cast(Op); |
0 |
| 5948 |
if (!CN) |
0 |
5948 |
if (!CN) |
0 |
| 5949 |
return SDValue(); |
0 |
5949 |
return SDValue(); |
0 |
| 5950 |
|
--- |
5950 |
|
--- |
| 5951 |
// Copy operands before the one being expanded. |
--- |
5951 |
// Copy operands before the one being expanded. |
--- |
| 5952 |
SmallVector NewOps; |
0 |
5952 |
SmallVector NewOps; |
0 |
| 5953 |
for (unsigned I = 0; I < OpNo; I++) |
0 |
5953 |
for (unsigned I = 0; I < OpNo; I++) |
0 |
| 5954 |
NewOps.push_back(N->getOperand(I)); |
0 |
5954 |
NewOps.push_back(N->getOperand(I)); |
0 |
| 5955 |
|
--- |
5955 |
|
--- |
| 5956 |
EVT Ty = Op.getValueType(); |
0 |
5956 |
EVT Ty = Op.getValueType(); |
0 |
| 5957 |
SDLoc DL = SDLoc(N); |
0 |
5957 |
SDLoc DL = SDLoc(N); |
0 |
| 5958 |
if (CN->getConstantIntValue()->getValue().getActiveBits() < 64) { |
0 |
5958 |
if (CN->getConstantIntValue()->getValue().getActiveBits() < 64) { |
0 |
| 5959 |
NewOps.push_back( |
0 |
5959 |
NewOps.push_back( |
0 |
| 5960 |
DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64)); |
0 |
5960 |
DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64)); |
0 |
| 5961 |
NewOps.push_back(DAG.getTargetConstant(CN->getZExtValue(), DL, Ty)); |
0 |
5961 |
NewOps.push_back(DAG.getTargetConstant(CN->getZExtValue(), DL, Ty)); |
0 |
| 5962 |
} else { |
--- |
5962 |
} else { |
--- |
| 5963 |
// FIXME: https://github.com/llvm/llvm-project/issues/55609 |
--- |
5963 |
// FIXME: https://github.com/llvm/llvm-project/issues/55609 |
--- |
| 5964 |
return SDValue(); |
0 |
5964 |
return SDValue(); |
0 |
| 5965 |
} |
--- |
5965 |
} |
--- |
| 5966 |
|
--- |
5966 |
|
--- |
| 5967 |
// Copy remaining operands. |
--- |
5967 |
// Copy remaining operands. |
--- |
| 5968 |
for (unsigned I = OpNo + 1; I < N->getNumOperands(); I++) |
0 |
5968 |
for (unsigned I = OpNo + 1; I < N->getNumOperands(); I++) |
0 |
| 5969 |
NewOps.push_back(N->getOperand(I)); |
0 |
5969 |
NewOps.push_back(N->getOperand(I)); |
0 |
| 5970 |
|
--- |
5970 |
|
--- |
| 5971 |
SDValue NewNode = DAG.getNode(N->getOpcode(), DL, N->getVTList(), NewOps); |
0 |
5971 |
SDValue NewNode = DAG.getNode(N->getOpcode(), DL, N->getVTList(), NewOps); |
0 |
| 5972 |
|
--- |
5972 |
|
--- |
| 5973 |
for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++) |
0 |
5973 |
for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++) |
0 |
| 5974 |
ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum)); |
0 |
5974 |
ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum)); |
0 |
| 5975 |
|
--- |
5975 |
|
--- |
| 5976 |
return SDValue(); // Signal that we have replaced the node already. |
0 |
5976 |
return SDValue(); // Signal that we have replaced the node already. |
0 |
| 5977 |
} |
0 |
5977 |
} |
0 |
| 5978 |
|
--- |
5978 |
|
--- |